Merge branch 'main' of https://github.com/nfdi4plants/DataHUB

Dockerfile

@@ -1,4 +1,4 @@
-ARG VERSION=latest
+ARG VERSION=17.5.5-ee.0
 FROM gitlab/gitlab-ee:$VERSION
 
 # re-set VERSION as its cleared after a FROM directive

README.md

@@ -4,7 +4,7 @@ This repository contains the Dockerfile and the necessary scripts and patches to
 
 Since the on-premise infrastructures vary greatly, we provide various examples of docker-compose.yml file in this README as a quick start. In case of assistance with more exotic setups, please contact us or write an issue and we will glady provide assistance.
 
-Note that in these examples, the `data` folder in the current directory is used to mount volumes for different aspects of the configuration & the data. For testing, it is enough to use a local folder. For production, it is recommended to use some kind of network storage or use storage volumes of your virtualisation solution as these usually snapshotted/backed up in some way.
+Note that in these examples, the `data` folder in the current directory is used to mount volumes for different aspects of the configuration & the data. For testing, it is enough to use a local folder. For production, it is recommended to use some kind of network storage or use storage volumes of your virtualisation solution as these are usually snapshotted/backed up in some way.
 
 For background information on the concept of the DataHUB see our [poster contribution to the 2023 CoRDI conference](https://doi.org/10.5281/zenodo.10021181), for the data publication workflow in conjunction with InvenioRDM see the IWSG conference paper.
 
@@ -17,8 +17,6 @@ It is important to note here, that the Enterprise Edition of GitLab is free and
 ## Quick start docker-compose file
 
 ```
-version: '3.9'
-
 services:
   web:
     image: ghcr.io/nfdi4plants/datahub:main
@@ -309,3 +307,30 @@ A job called "quality_report_generator" will be executed, which in turn will exe
 
 **Note:** The validation packages themselves are still a work in progress. Contact us if you need more information about which package would be available for testing.
 
+# DataHUB Update
+
+To upgrade your DataHUB, follow these steps:
+
+1. **Check Your GitLab Version**  
+   Find your current GitLab version in your instance under **Admin Area > Overview**.
+   These GitLab versions correspond to the DataHUB versions.
+
+2. **Determine the Upgrade Path**  
+   Use the [GitLab Upgrade Path](https://gitlab-com.gitlab.io/support/toolbox/upgrade-path/) to identify required intermediate/target versions.
+
+   Alternatively, you can also follow all upgrades up to the latest one listed [here](https://github.com/nfdi4plants/DataHUB/pkgs/container/datahub/versions?filters%5Bversion_type%5D=tagged).
+   There should be at least one working upgrade path for each version, please let us know if something is missing.
+
+3. **Pull the Matching DataHUB Image**  
+   DataHUB Docker images are tagged with the **exact DataHUB version** they support. Update your compose file with the next available DataHUB version needed in your upgrade path.
+   ```bash
+    services:
+      web:
+        image: ghcr.io/nfdi4plants/datahub:<release>
+   ```  
+   Replace `<release>` with your target GitLab version (e.g., `16.10.1-ee.0`).  
+   Check available versions/tags:  
+   [GitHub Container Registry for DataHUB](https://github.com/nfdi4plants/DataHUB/pkgs/container/datahub).  
+
+4. **Deploy the Upgraded Image**  
+   Follow your existing deployment workflow to apply the new image.

scripts/15.11.13-ee.0/patch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+
+for patch in /scripts/patches/*.patch; do
+	patch -p0 < "$patch"
+done
+
+for script in /scripts/patches/*.sh; do
+	"$script"
+done

scripts/15.11.13-ee.0/patches/_head.html.haml.patch

@@ -0,0 +1,9 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/projects/_head.html.haml	2024-01-17 13:51:44.978361699 +0000
++++ /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/projects/_head.html.haml	2024-01-17 13:51:14.477881902 +0000
+@@ -1,5 +1,5 @@
+ - breadcrumb_title _("Projects")
+-- page_title _("Explore projects")
++- page_title _("Explore ARCs")
+
+ = render_dashboard_ultimate_trial(current_user)
+

scripts/15.11.13-ee.0/patches/add-arc-js.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+set -- /opt/gitlab/embedded/service/gitlab-rails/public/assets/webpack/pages.projects.imports.new.*.chunk.js
+
+# exit if above's glob matches nothing or more than 1 file
+[ -e "$1" ] && [ "$#" = "1" ]
+
+# the mighty "patch"
+sed -Ei.bak \
+	's/sample:(\{text:Object\([^)]+\))/arc:\1("ProjectTemplates|ARC"),icon:".template-option .icon-gitlab_logo"},&/' \
+	"$1"
+
+# needs to be gzip'ed too
+gzip -kf "$1"
+
+# print diffs to see what was done
+diff -u "$1" "${1}.bak" || true

scripts/15.11.13-ee.0/patches/lfs_token.rb.patch

@@ -0,0 +1,11 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 15:32:49.679168014 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 12:52:34.978437027 +0200
+@@ -14,7 +14,7 @@
+
+     include LfsTokenHelper
+
+-    DEFAULT_EXPIRE_TIME = 7200 # Default value 2 hours
++    DEFAULT_EXPIRE_TIME = 86400 # Default value was 2 hours (=7200)
+
+     attr_accessor :actor
+

scripts/15.11.13-ee.0/patches/project_template.rb.patch

@@ -0,0 +1,10 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 15:34:11.873379730 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 12:52:34.981770257 +0200
+@@ -53,6 +53,7 @@
+       # rubocop:disable Metrics/AbcSize
+       def localized_templates_table
+         [
++          ProjectTemplate.new('arc', 'ARC', _('Includes DataPLANT ARC structure'), 'https://gitlab.com/gitlab-org/project-templates/rails', '/assets/dataplant-arc.png'),
+           ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'),
+           ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'),
+           ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'),

scripts/16.3.9-ee.0/patch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+
+for patch in /scripts/patches/*.patch; do
+	patch -p0 < "$patch"
+done
+
+for script in /scripts/patches/*.sh; do
+	"$script"
+done

scripts/16.3.9-ee.0/patches/_head.html.haml.patch

@@ -0,0 +1,9 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/projects/_head.html.haml	2024-01-17 13:51:44.978361699 +0000
++++ /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/projects/_head.html.haml	2024-01-17 13:51:14.477881902 +0000
+@@ -1,5 +1,5 @@
+ - breadcrumb_title _("Projects")
+-- page_title _("Explore projects")
++- page_title _("Explore ARCs")
+
+ = render_dashboard_ultimate_trial(current_user)
+

scripts/16.3.9-ee.0/patches/add-arc-js.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+set -- /opt/gitlab/embedded/service/gitlab-rails/public/assets/webpack/pages.projects.imports.new.*.chunk.js
+
+# exit if above's glob matches nothing or more than 1 file
+[ -e "$1" ] && [ "$#" = "1" ]
+
+# the mighty "patch"
+sed -Ei.bak \
+	's/sample:(\{text:Object\([^)]+\))/arc:\1("ProjectTemplates|ARC"),icon:".template-option .icon-gitlab_logo"},&/' \
+	"$1"
+
+# needs to be gzip'ed too
+gzip -kf "$1"
+
+# print diffs to see what was done
+diff -u "$1" "${1}.bak" || true

scripts/16.3.9-ee.0/patches/lfs_token.rb.patch

@@ -0,0 +1,11 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 15:32:49.679168014 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 12:52:34.978437027 +0200
+@@ -14,7 +14,7 @@
+
+     include LfsTokenHelper
+
+-    DEFAULT_EXPIRE_TIME = 7200 # Default value 2 hours
++    DEFAULT_EXPIRE_TIME = 86400 # Default value was 2 hours (=7200)
+
+     attr_accessor :actor
+

scripts/16.3.9-ee.0/patches/project_template.rb.patch

@@ -0,0 +1,10 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 15:34:11.873379730 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 12:52:34.981770257 +0200
+@@ -53,6 +53,7 @@
+       # rubocop:disable Metrics/AbcSize
+       def localized_templates_table
+         [
++          ProjectTemplate.new('arc', 'ARC', _('Includes DataPLANT ARC structure'), 'https://gitlab.com/gitlab-org/project-templates/rails', '/assets/dataplant-arc.png'),
+           ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'),
+           ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'),
+           ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'),

scripts/16.7.10-ee.0/patch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+
+for patch in /scripts/patches/*.patch; do
+	patch -p0 < "$patch"
+done
+
+for script in /scripts/patches/*.sh; do
+	"$script"
+done

scripts/16.7.10-ee.0/patches/_head.html.haml.patch

@@ -0,0 +1,11 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/_head.html.haml	2023-04-04 15:26:37.813601385 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/_head.html.haml	2023-04-04 14:03:00.950212922 +0200
+@@ -1,6 +1,6 @@
+ .explore-title.text-center
+   %h2
+-    = _("Explore GitLab")
++    = _("Explore DataPLANT")
+   %p.lead
+-    = _("Discover projects, groups and snippets. Share your projects with others")
++    = _("Discover ARCs, projects and groups. Share and contribute to ARCs with the DataPLANT community.")
+   %br

scripts/16.7.10-ee.0/patches/add-arc-js.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+set -- /opt/gitlab/embedded/service/gitlab-rails/public/assets/webpack/pages.projects.imports.new.*.chunk.js
+
+# exit if above's glob matches nothing or more than 1 file
+[ -e "$1" ] && [ "$#" = "1" ]
+
+# the mighty "patch"
+sed -Ei.bak \
+	's/sample:(\{text:Object\([^)]+\))/arc:\1("ProjectTemplates|ARC"),icon:".template-option .icon-gitlab_logo"},&/' \
+	"$1"
+
+# needs to be gzip'ed too
+gzip -kf "$1"
+
+# print diffs to see what was done
+diff -u "$1" "${1}.bak" || true

scripts/16.7.10-ee.0/patches/lfs_token.rb.patch

@@ -0,0 +1,11 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 15:32:49.679168014 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 12:52:34.978437027 +0200
+@@ -14,7 +14,7 @@
+
+     include LfsTokenHelper
+
+-    DEFAULT_EXPIRE_TIME = 7200 # Default value 2 hours
++    DEFAULT_EXPIRE_TIME = 86400 # Default value was 2 hours (=7200)
+
+     attr_accessor :actor
+

scripts/16.7.10-ee.0/patches/project_template.rb.patch

@@ -0,0 +1,10 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 15:34:11.873379730 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 12:52:34.981770257 +0200
+@@ -53,6 +53,7 @@
+       # rubocop:disable Metrics/AbcSize
+       def localized_templates_table
+         [
++          ProjectTemplate.new('arc', 'ARC', _('Includes DataPLANT ARC structure'), 'https://gitlab.com/gitlab-org/project-templates/rails', '/assets/dataplant-arc.png'),
+           ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'),
+           ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'),
+           ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'),

scripts/17.5.5-ee.0/patch.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -ex
+
+find /opt/gitlab/embedded/service/gitlab-rails/app/views/profiles/
+ls -la /opt/gitlab/embedded/service/gitlab-rails/app/views/profiles/accounts/show.html.haml
+ls -la /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb 
+
+for patch in /scripts/patches/*.patch; do
+	patch -p0 < "$patch"
+done
+
+
+for script in /scripts/patches/*.sh; do
+	"$script"
+done

scripts/17.5.5-ee.0/patches/_head.html.haml.patch

@@ -0,0 +1,9 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/projects/_head.html.haml	2024-01-17 13:51:44.978361699 +0000
++++ /opt/gitlab/embedded/service/gitlab-rails/app/views/explore/projects/_head.html.haml	2024-01-17 13:51:14.477881902 +0000
+@@ -1,5 +1,5 @@
+ - breadcrumb_title _("Projects")
+-- page_title _("Explore projects")
++- page_title _("Explore ARCs")
+
+ = render_dashboard_ultimate_trial(current_user)
+

scripts/17.5.5-ee.0/patches/add-arc-js.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+set -- /opt/gitlab/embedded/service/gitlab-rails/public/assets/webpack/pages.projects.imports.new.*.chunk.js
+
+# exit if above's glob matches nothing or more than 1 file
+[ -e "$1" ] && [ "$#" = "1" ]
+
+# the mighty "patch"
+sed -Ei.bak \
+	's/sample:(\{text:Object\([^)]+\))/arc:\1("ProjectTemplates|ARC"),icon:".template-option .icon-gitlab_logo"},&/' \
+	"$1"
+
+# needs to be gzip'ed too
+gzip -kf "$1"
+
+# print diffs to see what was done
+diff -u "$1" "${1}.bak" || true

scripts/17.5.5-ee.0/patches/carrierwave_s3_encryption_headers_patch.rb.patch

@@ -0,0 +1,11 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/config/initializers/carrierwave_s3_encryption_headers_patch.rb	2024-07-10 10:16:24.201574831 +0000
++++ /opt/gitlab/embedded/service/gitlab-rails/config/initializers/carrierwave_s3_encryption_headers_patch.rb	2024-07-10 10:16:57.405284779 +0000
+@@ -25,7 +25,7 @@
+           if ::Feature.enabled?(:s3_multithreaded_uploads, type: :ops) && fog_provider == 'AWS'
+             # AWS SDK uses 10 threads by default and a multipart chunk size of 10 MB
+             file.concurrency = 10
+-            file.multipart_chunk_size = 10485760
++            file.multipart_chunk_size = 10485760 / 10 * 1024 # 1 GiB
+             file.copy(@uploader.fog_directory, new_path, copy_to_options)
+           else
+             # Some Fog providers may issue a GET request (https://github.com/fog/fog-google/issues/512)

scripts/17.5.5-ee.0/patches/direct_upload.rb.patch

@@ -0,0 +1,19 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/object_storage/direct_upload.rb	2024-07-10 10:14:00.134830913 +0000
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/object_storage/direct_upload.rb	2024-08-21 15:01:53.519335954 +0000
+@@ -16,7 +16,7 @@
+   class DirectUpload
+     include Gitlab::Utils::StrongMemoize
+
+-    TIMEOUT = 4.hours
++    TIMEOUT = 48.hours
+     EXPIRE_OFFSET = 15.minutes
+
+     MAXIMUM_MULTIPART_PARTS = 100
+@@ -133,6 +133,7 @@
+     end
+
+     def use_workhorse_s3_client?
++      return false
+       return false unless config.use_iam_profile? || config.consolidated_settings?
+       # The Golang AWS SDK does not support V2 signatures
+       return false unless credentials.fetch(:aws_signature_version, 4).to_i >= 4

scripts/17.5.5-ee.0/patches/lfs_token.rb.patch

@@ -0,0 +1,11 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 15:32:49.679168014 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/lfs_token.rb	2023-04-04 12:52:34.978437027 +0200
+@@ -14,7 +14,7 @@
+
+     include LfsTokenHelper
+
+-    DEFAULT_EXPIRE_TIME = 7200 # Default value 2 hours
++    DEFAULT_EXPIRE_TIME = 86400 # Default value was 2 hours (=7200)
+
+     attr_accessor :actor
+

scripts/17.5.5-ee.0/patches/project_template.rb.patch

@@ -0,0 +1,10 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 15:34:11.873379730 +0200
++++ /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/project_template.rb	2023-04-04 12:52:34.981770257 +0200
+@@ -53,6 +53,7 @@
+       # rubocop:disable Metrics/AbcSize
+       def localized_templates_table
+         [
++          ProjectTemplate.new('arc', 'ARC', _('Includes DataPLANT ARC structure'), 'https://gitlab.com/gitlab-org/project-templates/rails', '/assets/dataplant-arc.png'),
+           ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'),
+           ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'),
+           ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'),

scripts/17.5.5-ee.0/patches/show.html.haml.patch

@@ -0,0 +1,36 @@
+--- /opt/gitlab/embedded/service/gitlab-rails/app/views/profiles/accounts/show.html.haml	2025-01-08 10:40:34.000000000 +0000
++++ /opt/gitlab/embedded/service/gitlab-rails/app/views/profiles/accounts/show.html.haml	2025-02-04 13:42:23.946661640 +0000
+@@ -8,32 +8,6 @@
+     - c.with_body do
+       = s_('Profiles|Some options are unavailable for LDAP accounts')
+
+-- if params[:two_factor_auth_enabled_successfully]
+-  = render Pajamas::AlertComponent.new(variant: :success,
+-    alert_options: { class: 'gl-my-5' },
+-    close_button_options: { class: 'js-close-2fa-enabled-success-alert' }) do |c|
+-    - c.with_body do
+-      = html_escape(_('You have set up 2FA for your account! If you lose access to your 2FA device, you can use your recovery codes to access your account. Alternatively, if you upload an SSH key, you can %{anchorOpen}use that key to generate additional recovery codes%{anchorClose}.')) % { anchorOpen: '<a href="%{href}">'.html_safe % { href: help_page_path('user/profile/account/two_factor_authentication_troubleshooting.md', anchor: 'generate-new-recovery-codes-using-ssh') }, anchorClose: '</a>'.html_safe }
+-
+-.settings-section.js-search-settings-section
+-  .settings-sticky-header
+-    .settings-sticky-header-inner
+-      %h4.gl-my-0
+-        = s_('Profiles|Two-factor authentication')
+-  %p.gl-text-secondary
+-    = s_("Profiles|Increase your account's security by enabling two-factor authentication (2FA).")
+-  %div
+-    %p
+-      %span.gl-font-bold
+-        #{_('Status')}:
+-      #{current_user.two_factor_enabled? ? _('Enabled') : _('Disabled')}
+-    - if current_user.two_factor_enabled?
+-      = render Pajamas::ButtonComponent.new(variant: :confirm, href: profile_two_factor_auth_path) do
+-        = _('Manage two-factor authentication')
+-    - else
+-      = render Pajamas::ButtonComponent.new(variant: :confirm, href: profile_two_factor_auth_path, button_options: { data: { testid: 'enable-2fa-button' }}) do
+-        = _('Enable two-factor authentication')
+-
+ - if display_providers_on_profile?
+   .settings-section.js-search-settings-section
+     .settings-sticky-header
+