diff --git a/.env b/.env new file mode 100644 index 000000000..d25101b9c --- /dev/null +++ b/.env @@ -0,0 +1,170 @@ +# Environment variables defined in this file apply to both the Makefile and to +# docker-compose.yml +# +# Due to restrictions in the `env-file` format we cannot specify multi-line +# values for environment variables. For this reason the environment +# variables are set on service definitions in the docker-compose.*.yml files, +# rather than defined in `env-file` files. + +# Determines which docker-compose file(s) will be used for the `drupal` service. +# See documentation for more details. +ENVIRONMENT=demo + +# Enable this to generate a docker-compose file that uses secrets. +# If you're running staging, CI, or production, set to true. +USE_SECRETS=true + +############################################################################### +# Environment variables specific to composer. +############################################################################### + +COMPOSE_HTTP_TIMEOUT=480 + +# Also used for naming services in traefik as well as defining network alias and urls. +# For example the `drupal` service will be found at `islandora.${COMPOSE_PROJECT_NAME}.${DRUPAL_SITE_HOST}`. +# See https://docs.docker.com/compose/reference/envvars/ +COMPOSE_PROJECT_NAME=isle-dc + +# Allows building custom image with buildkit. +COMPOSE_DOCKER_CLI_BUILD=1 +DOCKER_BUILDKIT=1 + +# Dockerfile to use when building the custom project. +PROJECT_DRUPAL_DOCKERFILE=Dockerfile + +# Includes `traefik` as a service, if false assume we are sharing a traefik +# from another project. +INCLUDE_TRAEFIK_SERVICE=true + +# Should we use ACME to generate a SSL Certificate +USE_ACME=false +# Specify email to tie SSL Certificate to with ACME provider +ACME_EMAIL=your-email@example.com + +# Includes `watchtower` as a service. +INCLUDE_WATCHTOWER_SERVICE=false + +# Includes `etcd` as a service. +INCLUDE_ETCD_SERVICE=false + +# Includes `code-server` as a service. +INCLUDE_CODE_SERVER_SERVICE=false + +# Choose which database backend to use: mariadb/postgresql +# Services that only support MySQL (Matomo) do not have the option to change. +# Also at this time not all Drupal modules work with PostgresSQL, it is provided +# as option here so it can be tested and fixed at some later date. +DRUPAL_DATABASE_SERVICE=mariadb +FCREPO_DATABASE_SERVICE=mariadb + +# Repository to use for pulling isle-buildkit images, change to `local` +# To use images you have built locally with isle-buildkit, or use your +# custom docker registry if you have set up one. +# +REPOSITORY=islandora + +# The version of the isle-buildkit images, non isle-buildkit images have +# their versions specified explicitly in their respective docker-compose files. +TAG=1.0.10 + +############################################################################### +# Exposed Containers & Ports +############################################################################### + +# Expose Cantaloupe at ${DOMAIN}/cantaloupe +EXPOSE_CANTALOUPE=true + +# Expose Matomo at ${DOMAIN}/matomo +EXPOSE_MATOMO=false + +# Expose Drupal at ${DOMAIN} +EXPOSE_DRUPAL=true + +# Expose MySQL over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_MYSQL=false +MYSQL_PORT=3306 + +# Expose Postgres over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_POSTGRES=false +POSTGRES_PORT=5432 + +# Expose the Traefik dashboard over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_TRAEFIK_DASHBOARD=false +TRAEFIK_DASHBOARD_PORT=8080 + +# Expose Fedora over the given port - Necessary for Canteloupe +EXPOSE_FEDORA=true +FEDORA_PORT=8081 + +# Expose Blazegraph over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_BLAZEGRAPH=true +BLAZEGRAPH_PORT=8082 + +# Expose Activemq over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_ACTIVEMQ=false +ACTIVEMQ_PORT=8161 + +# Expose SOLR over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_SOLR=true +SOLR_PORT=8983 + +# Expose Code Server over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_CODE_SERVER=false +CODE_SERVER_PORT=8443 + +############################################################################### +# Global Environment variables +############################################################################### + +DOMAIN=islandora.traefik.me +SITE=https://${DOMAIN} +DISABLE_SYN=false +FEDORA_6=true +RESTART_POLICY=unless-stopped + +# PHP variables +PHP_MEMORY_LIMIT=5G +PHP_POST_MAX_SIZE=5G +PHP_UPLOAD_MAX_FILESIZE=5G +PHP_MAX_EXECUTION_TIME=30 + +# If you're just demoing or are starting from scratch, use this. +INSTALL_EXISTING_CONFIG=true +DRUPAL_INSTALL_PROFILE=standard + +# If you're installing from an existing codebase, uncomment this +#INSTALL_EXISTING_CONFIG=true +#DRUPAL_INSTALL_PROFILE=minimal + +# If you're experiencing timeouts in Alpaca, consider bumping these +# values for each connector. They are in milliseconds (1000 = 1 second). +ALPACA_OCR_TIMEOUT=300000 +ALPACA_FITS_TIMEOUT=300000 +ALPACA_HOUDINI_TIMEOUT=300000 +ALPACA_HOMARUS_TIMEOUT=300000 + +# Set Memory Limit for each container. +ACTIVEMQ_MEMORY_LIMIT=2G +ALPACA_MEMORY_LIMIT=2G +BLAZEGRAPH_MEMORY_LIMIT=5G +CANTALOUPE_MEMORY_LIMIT=5G +CRAYFITS_MEMORY_LIMIT=1G +DRUPAL_MEMORY_LIMIT=5G +FCREPO_MEMORY_LIMIT=5G +FITS_MEMORY_LIMIT=5G +HOMARUS_MEMORY_LIMIT=1G +HOUDINI_MEMORY_LIMIT=1G +HYPERCUBE_MEMORY_LIMIT=512M +IDE_MEMORY_LIMIT=5G +MARIADB_MEMORY_LIMIT=1G +MATOMO_MEMORY_LIMIT=1G +MILLINER_MEMORY_LIMIT=1G +RECAST_MEMORY_LIMIT=1G +SOLR_MEMORY_LIMIT=8G +TRAEFIK_MEMORY_LIMIT=8G +WATCHTOWER_MEMORY_LIMIT=2G + +# Configuration to enable the custom delegate script for Cantaloupe +CANTALOUPE_DELEGATE_SCRIPT_ENABLED=false +CANTALOUPE_DELEGATE_SCRIPT_PATHNAME=/opt/tomcat/bin/delegates.rb +CANTALOUPE_HTTPSOURCE_LOOKUP_STRATEGY=BasicLookupStrategy diff --git a/.github/README.md b/.github/README.md new file mode 120000 index 000000000..055b6875e --- /dev/null +++ b/.github/README.md @@ -0,0 +1 @@ +../IDC.md \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..96af22d38 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,191 @@ +name: CI +on: + push: + branches: + - main + - development + tags: + - v* + pull_request: + branches: + - main + - development + +jobs: + build: + name: build + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.excludes-matrix.outputs.matrix }} + steps: + # Check out current commit + - name: Checkout + uses: actions/checkout@v3 + + # Make sure buildkit is enabled + - name: Enable buildkit + shell: bash + run: | + echo '{"experimental": "enabled"}' > ~/.docker/config.json + + # Build and Export image for push to github container registry + - name: Build and export drupal image + run: make static-drupal-image-export + + # Build and run the static environment + - name: Build Static + run: make static-docker-compose.yml up + + - uses: actions/upload-artifact@v2 + with: + name: drupal-image + path: images + + - name: Generate Test Matrix + id: test-matrix + uses: jhu-idc/idc-matrixgen@1.0.0 + with: + key: suite + + - name: Exclude nightly jobs + id: excludes-matrix + uses: jhu-idc/idc-matrixgen@1.0.0 + with: + key: suite + glob: '*nightly*.sh' + exclude: true + matrix: ${{ steps.test-matrix.outputs.matrix }} + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: failure() + test: + name: test + runs-on: ubuntu-latest + if: false # Don/Tim: disable testing for now + needs: build + strategy: + matrix: ${{ fromJSON(needs.build.outputs.matrix) }} + fail-fast: false + steps: + # Check out current commit + - name: Checkout + uses: actions/checkout@v3 + + - name: Download Drupal Image + uses: actions/download-artifact@v2 + with: + name: drupal-image + path: images + + - name: Load Drupal Image + run: docker load < images/static-drupal.tar + + - name: Use static docker-compose file, to test static image + run: make static-docker-compose.yml + + - name: Up + run: make up + + - name: test + run: | + mkdir -p end-to-end/reports + chmod a+rwx end-to-end/reports + make test test=${{ matrix.suite }} + - name: reports + if: always() + run: | + mkdir -p end-to-end/reports/${{ matrix.suite }} + docker ps + docker logs idc_drupal_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/drupal-${{ matrix.suite }}.log + docker logs idc_alpaca_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/alpaca-${{ matrix.suite }}.log + docker logs idc_homarus_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/homarus-${{ matrix.suite }}.log + docker logs idc_crayfits_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/crayfits-${{ matrix.suite }}.log + docker logs idc_fits_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/fits-${{ matrix.suite }}.log + docker logs idc_houdini_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/houdini-${{ matrix.suite }}.log + docker logs idc_dlq_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/dlq-${{ matrix.suite }}.log + - name: upload reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: reports-${{ matrix.suite }} + path: end-to-end/reports/${{ matrix.suite }} + - name: upload screenshots + if: always() + uses: actions/upload-artifact@v2 + with: + name: reports-screenshots + path: end-to-end/reports/screenshots + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + MATRIX_CONTEXT: ${{ toJson(matrix) }} # required + if: failure() + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: [ build ] # Tim, Don: removing test dependency + env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + DOCKER_PUSH_REPOSITORY: ${{secrets.DOCKER_PUSH_REPOSITORY}} + steps: + # Check out current commit + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Docker Login + if: ${{ github.event_name != 'pull_request' && env.DOCKER_USER }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ env.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Download Drupal Image + if: ${{ github.event_name != 'pull_request' && env.DOCKER_USER }} + uses: actions/download-artifact@v2 + with: + name: drupal-image + path: images + + - name: Load Drupal Image + if: ${{ github.event_name != 'pull_request' && env.DOCKER_USER }} + run: docker load < images/static-drupal.tar + + - name: Use static docker-compose file, to deploy image + run: make static-docker-compose.yml + + # The primary purpose of this step is to make sure all the non-drupal images are + # downloaded. + - name: Load the irest of the stack + run: docker-compose up -d + + # Push docker images, if we are on the appropriate branch or tag + - name: Docker Push + if: ${{ github.event_name != 'pull_request' && env.DOCKER_USER }} + run: sh tag_and_push_images.sh + + # Experimental way to delete artifact + - name: Delete Temporary Images + uses: geekyeggo/delete-artifact@v1 + with: + name: drupal-image + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: failure() diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 000000000..af8d2ea82 --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,22 @@ +# This workflow was added by CodeSee. Learn more at https://codesee.io/ +# This is v2.0 of this workflow file +on: + push: + branches: + - development + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee + +permissions: read-all + +jobs: + codesee: + runs-on: ubuntu-latest + continue-on-error: true + name: Analyze the repo with CodeSee + steps: + - uses: Codesee-io/codesee-action@v2 + with: + codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 000000000..a55fb1a29 --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,95 @@ +name: Dependency Diff +on: + pull_request: + branches: [development] +jobs: + base: + name: Calculate and Upload Base Dependencies + # This should always be true for pull request events + if: ${{ github.base_ref != null }} + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + DEP_FILENAME: deps.${{ github.event.pull_request.base.sha }} + outputs: + dep-filename: ${{ steps.base-dep-filename.outputs.dep_filename }} + steps: + - name: Checkout Base Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + - name: Start IDC + run: make up + - name: Capture Dependencies + run: docker-compose exec -T drupal composer show > ${{ env.DEP_FILENAME }} + - name: Upload + uses: actions/upload-artifact@v2 + with: + name: base-dependencies + path: ${{ env.DEP_FILENAME }} + - name: Set Outputs + id: base-dep-filename + run: echo "::set-output name=dep_filename::${{ env.DEP_FILENAME }}" + pr: + name: Calculate and Upload PR Dependencies + # This should always be true for pull request events + if: ${{ github.head_ref != null }} + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + DEP_FILENAME: deps.pr-${{ github.event.number }}.${{ github.event.pull_request.head.sha }} + outputs: + dep-filename: ${{ steps.pr-dep-filename.outputs.dep_filename }} + steps: + - name: Checkout PR Branch + uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: 0 + - name: Start IDC + run: make up + - name: Capture Dependencies + run: docker-compose exec -T drupal composer show > ${{ env.DEP_FILENAME }} + - name: Upload + uses: actions/upload-artifact@v2 + with: + name: pr-dependencies + path: ${{ env.DEP_FILENAME }} + - name: Set Outputs + id: pr-dep-filename + run: echo "::set-output name=dep_filename::${{ env.DEP_FILENAME }}" + diff: + name: Create and Upload Diff + needs: [base, pr] + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + DIFF_FILENAME: out.diff + steps: + - name: Download Base Deps + uses: actions/download-artifact@v2 + with: + name: base-dependencies + - name: Download PR Deps + uses: actions/download-artifact@v2 + with: + name: pr-dependencies + - name: Calculate diff + # ignore exit code from git diff + shell: bash {0} + run: | + echo "Generating diff between ${{ needs.base.outputs.dep-filename }} and ${{ needs.pr.outputs.dep-filename }} to ${DIFF_FILENAME}" + echo "Dependency diff between development base branch ${{ github.base_ref }} (${{ github.event.pull_request.base.sha }}) and PR branch ${{ github.head_ref }} (${{ github.event.pull_request.head.sha }}):" >> ${DIFF_FILENAME} + echo '```diff' >> ${DIFF_FILENAME} + git diff --no-index -w ${{ needs.base.outputs.dep-filename }} ${{ needs.pr.outputs.dep-filename }} >> ${DIFF_FILENAME} + if [ $? -gt 0 ] ; then \ + echo '```' >> ${DIFF_FILENAME} ; \ + else \ + echo "> This PR has no dependency differences with the base branch" > ${DIFF_FILENAME} ; \ + fi + - name: Comment on PR + uses: machine-learning-apps/pr-comment@1.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + path: ${{ env.DIFF_FILENAME }} diff --git a/.github/workflows/deploy_test.yml b/.github/workflows/deploy_test.yml new file mode 100644 index 000000000..44521831b --- /dev/null +++ b/.github/workflows/deploy_test.yml @@ -0,0 +1,62 @@ +name: Deploy and Rollback + +on: + push: + branches: + - community_version + workflow_dispatch: + inputs: + action: + description: 'Action (deploy/latest or rollback)' + required: true + default: 'deploy/latest' + +jobs: + build-and-deploy: + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.action == 'deploy/latest') + name: build and deploy + runs-on: + - self-hosted + - testing + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build JHU Image + run: make jhu_up + + - name: Add demo content + run: make jhu_demo_content + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() + + rollback: + if: github.event_name == 'workflow_dispatch' && github.event.inputs.action == 'rollback' + name: rollback + runs-on: + - self-hosted + - testing + steps: + # Add steps here for rolling back your deployment, e.g.: + - name: Checkout previous version + uses: actions/checkout@v3 + with: + ref: ${{ github.event.before }} + + # ... additional rollback steps ... + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..e914cf04b --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,112 @@ +name: Nightly +on: + schedule: + - cron: '5 4 * * *' + push: + branches: [ development ] + workflow_dispatch: +jobs: + build: + name: build + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.test-matrix.outputs.matrix }} + env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + steps: + # Check out current commit + - name: Checkout + uses: actions/checkout@v3 + + # Make sure buildkit is enabled + - name: Enable buildkit + shell: bash + run: | + echo '{"experimental": "enabled"}' > ~/.docker/config.json + + # Build and run the static environment + - name: Build Static + run: make static-docker-compose.yml up + + # Drush tests + - name: Drush test + run: docker-compose exec -T drupal drush -d status + + - name: Export image + run: make static-drupal-image-export + + - uses: actions/upload-artifact@v2 + with: + name: drupal-image + path: images + + - name: Generate Test Matrix + id: test-matrix + uses: jhu-idc/idc-matrixgen@1.0.0 + with: + key: suite + glob: '*nightly*.sh' + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: failure() + test: + name: test + runs-on: ubuntu-latest + needs: build + strategy: + matrix: ${{ fromJSON(needs.build.outputs.matrix) }} + fail-fast: false + steps: + # Check out current commit + - name: Checkout + uses: actions/checkout@v3 + + - name: Download Drupal Image + uses: actions/download-artifact@v2 + with: + name: drupal-image + path: images + + - name: Load Drupal Image + run: docker load < images/static-drupal.tar + + - name: Up + run: make up + + - name: test + run: | + mkdir -p end-to-end/reports + chmod a+rwx end-to-end/reports + make test test=${{ matrix.suite }} + - name: reports + if: always() + run: | + mkdir -p end-to-end/reports/${{ matrix.suite }} + docker ps + docker logs idc_drupal_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/drupal-${{ matrix.suite }}.log + docker logs idc_alpaca_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/alpaca-${{ matrix.suite }}.log + docker logs idc_homarus_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/homarus-${{ matrix.suite }}.log + docker logs idc_crayfits_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/crayfits-${{ matrix.suite }}.log + docker logs idc_fits_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/fits-${{ matrix.suite }}.log + docker logs idc_houdini_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/houdini-${{ matrix.suite }}.log + docker logs idc_dlq_1 2>&1 | tee end-to-end/reports/${{ matrix.suite }}/dlq-${{ matrix.suite }}.log + - name: upload reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: reports-${{ matrix.suite }} + path: end-to-end/reports/${{ matrix.suite }} + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + MATRIX_CONTEXT: ${{ toJson(matrix) }} # required + if: failure() diff --git a/.gitignore b/.gitignore index 10b3602ce..2a6a93d37 100644 --- a/.gitignore +++ b/.gitignore @@ -66,7 +66,8 @@ site *.retry .vscode config/proxy/acme.json -.env +secrets/live/* +islandora_workbench #=================== # Drupal and Mariadb files persistant storage @@ -82,6 +83,7 @@ docker-compose.yml #=================== # User overrides for environment variables. docker-compose.env.yml +docker-compose.override.yml #=================== # User overrides for building custom image from the codebase directory. @@ -90,3 +92,4 @@ Dockerfile #=================== # User provided certs for TLS. certs +acme diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e4e9e7ca..199cd90d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ That is great! In this case please send us a pull request as described in sectio ### Contribute code -Before you set out to contribute code you will need to have completed a [Contributor License Agreement](http://islandora.ca/sites/default/files/islandora_cla.pdf) or be covered by a [Corporate Contributor Licencse Agreement](http://islandora.ca/sites/default/files/islandora_ccla.pdf). The signed copy of the license agreement should be sent to +Before you set out to contribute code you will need to have completed a [Contributor License Agreement](http://islandora.ca/sites/default/files/islandora_cla.pdf) or be covered by a [Corporate Contributor License Agreement](http://islandora.ca/sites/default/files/islandora_ccla.pdf). The signed copy of the license agreement should be sent to _If you are interested in contributing code to ISLE 8 but do not know where to begin:_ @@ -105,4 +105,6 @@ TBD ## License Agreements -The Islandora Foundation requires that contributors complete a [Contributor License Agreement](http://islandora.ca/sites/default/files/islandora_cla.pdf) or be covered by a [Corporate Contributor Licencse Agreement](http://islandora.ca/sites/default/files/islandora_ccla.pdf). The signed copy of the license agreement should be sent to community@islandora.ca. This license is for your protection as a contributor as well as the protection of the Foundation and its users; it does not change your rights to use your own contributions for any other purpose. +The Islandora Foundation requires that contributors complete a [Contributor License Agreement](https://forms.gle/7YYShuaMfG7F94A87) or be covered by a [Corporate Contributor License Agreement](https://forms.gle/7YYShuaMfG7F94A87). This license is for your protection as a contributor as well as the protection of the Foundation and its users; it does not change your rights to use your own contributions for any other purpose. + +**To complete a Contributor License Agreement (Individual or Corporate), please click [**HERE**](https://forms.gle/kS6BKhaf5LBzNvj18) to be taken to the Google Form.** diff --git a/IDC-README.md b/IDC-README.md new file mode 100644 index 000000000..5db3461fb --- /dev/null +++ b/IDC-README.md @@ -0,0 +1,38 @@ +## To run +```shell +make jhu_up +``` + +## To stop +Stop without losing data. +```shell +make jhu_down +``` + +## To stop and reset +Stop and reset without removing codebase directory. +```shell +make jhu_clean +``` +## Editing the pages +Pages can be impacted by view modes, twig templates, context, and views. The following sections describe how to edit each of these parts. + +## How to edit frontpage +Parts: +- Twig template: `idc_ui_theme_boots/templates/pages/page--front.html.twig` +- View: +- View Mode: + +## How to edit collections/ page +Parts: +- Twig template: `idc_ui_theme_boots/templates/content/node--view--members.html.twig` +- View: Solr search content (Index Default Solr content index) `admin/structure/views/view/solr_search_content/edit/block_1` +- View Mode: Context > Collection `admin/structure/context/collection` + +## How to edit search bar on frontpage +Parts: +- Twig template: +- View: +- View Mode: Solr search content (Index Default Solr content index) `admin/structure/views/view/solr_search_content/edit/page_1` +- Block: `Exposed form: solr_search_content-page_1` + diff --git a/Makefile b/Makefile index c2a93ad22..5321abc9c 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,26 @@ ENV_FILE=$(shell \ fi; \ echo .env) +# handle sed -i differences (see https://stackoverflow.com/a/4247319/3204023 for more context) +ifeq ($(shell uname -s),Darwin) + SED_DASH_I=sed -i '' +else # GNU/Linux + SED_DASH_I=sed -i +endif + + +# If custom.makefile exists include it. +-include custom.Makefile + +# Checks to see if the path includes a space character. Intended to be a temporary fix. +ifneq (1,$(words $(CURDIR))) +$(error Containing path cannot contain space characters: '$(CURDIR)') +endif + # Include the sample.env so new values can be added with defaults without requiring # users to regenerate their .env files losing their changes. include sample.env include $(ENV_FILE) - # The site to operate on when using drush -l $(SITE) commands SITE?=default @@ -22,7 +37,11 @@ export EXTERNAL_SERVICES := etcd watchtower traefik # The minimal set of docker-compose files required to be able to run anything. -REQUIRED_SERIVCES ?= activemq alpaca blazegraph cantaloupe crayfish crayfits drupal fcrepo mariadb matomo solr +REQUIRED_SERVICES ?= activemq alpaca blazegraph cantaloupe crayfish crayfits drupal mariadb matomo solr + +ifeq ($(USE_SECRETS), true) + SECRETS := secrets +endif # Watchtower is an optional dependency, by default it is included. ifeq ($(INCLUDE_WATCHTOWER_SERVICE), true) @@ -34,15 +53,32 @@ ifeq ($(INCLUDE_TRAEFIK_SERVICE), true) TRAEFIK_SERVICE := traefik endif +# The service traefik may be optional if we are sharing one from another project. +ifeq ($(USE_ACME), true) + ACME := acme +endif + +# The service traefik may be optional if we are sharing one from another project. +ifeq ($(INCLUDE_CODE_SERVER_SERVICE), true) + CODE_SERVER_SERVICE := code-server +endif + # etcd is an optional dependency, by default it is not included. ifeq ($(INCLUDE_ETCD_SERVICE), true) ETCD_SERVICE := etcd endif +# etcd is an optional dependency, by default it is not included. +ifeq ($(FEDORA_6), true) + FCREPO_SERVICE := fcrepo6 +else + FCREPO_SERVICE := fcrepo +endif + # Some services can optionally depend on PostgreSQL. # Either way their environment variables get customized # depending on the database service they have choosen. -DATABASE_SERVICES ?= drupal.$(DRUPAL_DATABASE_SERVICE) fcrepo.$(FCREPO_DATABASE_SERVICE) crayfish.$(GEMINI_DATABASE_SERVICE) +DATABASE_SERVICES ?= drupal.$(DRUPAL_DATABASE_SERVICE) $(FCREPO_SERVICE).$(FCREPO_DATABASE_SERVICE) ifeq ($(DRUPAL_DATABASE_SERVICE), postgresql) DATABASE_SERVICES += postgresql @@ -52,33 +88,22 @@ ifeq ($(FCREPO_DATABASE_SERVICE), postgresql) DATABASE_SERVICES += postgresql endif -ifeq ($(GEMINI_DATABASE_SERVICE), postgresql) - DATABASE_SERVICES += postgresql -endif - # Sorts and removes duplicates. DATABASE_SERVICES := $(sort $(DATABASE_SERVICES)) -# Allows for customization of the environment variables inside of the containers. -# If it does not exist create it from docker-compose.sample.env.yml. -OVERRIDE_SERVICE_ENVIRONMENT_VARIABLES=$(shell \ - if [ ! -f docker-compose.env.yml ]; then \ - cp docker-compose.sample.env.yml docker-compose.env.yml; \ - fi; \ - echo env) - # The services to be run (order is important), as services can override one -# another. Traefik must be last if included as otherwise its network +# another. Traefik must be last if included as otherwise its network # definition for `gateway` will be overriden. -SERVICES := $(REQUIRED_SERIVCES) $(WATCHTOWER_SERVICE) $(ETCD_SERVICE) $(DATABASE_SERVICES) $(ENVIRONMENT) $(TRAEFIK_SERVICE) $(OVERRIDE_SERVICE_ENVIRONMENT_VARIABLES) +SERVICES := $(REQUIRED_SERVICES) $(FCREPO_SERVICE) $(WATCHTOWER_SERVICE) $(ETCD_SERVICE) $(DATABASE_SERVICES) $(ENVIRONMENT) $(SECRETS) $(CODE_SERVER_SERVICE) $(TRAEFIK_SERVICE) $(ACME) default: download-default-certs docker-compose.yml pull .SILENT: docker-compose.yml -docker-compose.yml: $(SERVICES:%=docker-compose.%.yml) .env - docker-compose $(SERVICES:%=-f docker-compose.%.yml) config > docker-compose.yml +docker-compose.yml: $(SERVICES:%=build/docker-compose/docker-compose.%.yml) .env + docker-compose $(SERVICES:%=-f build/docker-compose/docker-compose.%.yml) config > docker-compose.yml .PHONY: pull +## Fetches the latest images from the registry. pull: docker-compose.yml ifeq ($(REPOSITORY), local) # Only need to pull external services if using local images. @@ -88,194 +113,217 @@ else endif .PHONY: build +## Create Dockerfile from example if it does not exist. build: - # Create Dockerfile from example if it does not exist. if [ ! -f $(PROJECT_DRUPAL_DOCKERFILE) ]; then \ - cp $(CURDIR)/sample.Dockerfile $(PROJECT_DRUPAL_DOCKERFILE); \ + cp "$(CURDIR)/sample.Dockerfile" $(PROJECT_DRUPAL_DOCKERFILE); \ fi docker build -f $(PROJECT_DRUPAL_DOCKERFILE) -t $(COMPOSE_PROJECT_NAME)_drupal --build-arg REPOSITORY=$(REPOSITORY) --build-arg TAG=$(TAG) . -# Updates codebase folder to be owned by the host user and nginx group. -.PHONY: set-codebase-owner -.SILENT: set-codebase-owner -set-codebase-owner: - sudo find ./codebase -exec chown $(shell id -u):101 {} \; +.PHONY: set-files-owner +## Updates codebase folder to be owned by the host user and nginx group. +.SILENT: set-files-owner +set-files-owner: $(SRC) +ifndef SRC + $(error SRC is not set) +endif + sudo chown -R $(shell id -u):101 $(SRC) -# Creates required databases for drupal site(s) using environment variables. -.PHONY: databases -.SILENT: databases -databases: - docker-compose exec drupal with-contenv bash -lc "for_all_sites create_database" +.PHONY: drupal-database +## Creates required databases for drupal site(s) using environment variables. +.SILENT: drupal-database +drupal-database: + docker-compose exec -T drupal timeout 300 bash -c "while ! test -e /var/run/nginx/nginx.pid -a -e /var/run/php-fpm7/php-fpm7.pid; do sleep 1; done" + docker-compose exec -T drupal with-contenv bash -lc "for_all_sites create_database" -# Installs drupal site(s) using environment variables. .PHONY: install +## Installs drupal site(s) using environment variables. .SILENT: install -install: databases - docker-compose exec drupal with-contenv bash -lc "for_all_sites install_site" +install: drupal-database + docker-compose exec -T drupal with-contenv bash -lc "for_all_sites install_site" -# Updates settings.php according to the environment variables. .PHONY: update-settings-php +## Updates settings.php according to the environment variables. .SILENT: update-settings-php update-settings-php: - docker-compose exec drupal with-contenv bash -lc "for_all_sites update_settings_php" + docker-compose exec -T drupal with-contenv bash -lc "if [ ! -f /var/www/drupal/web/sites/default/settings.php ]; then cp /var/www/drupal/web/sites/default/default.settings.php /var/www/drupal/web/sites/default/settings.php; fi" + docker-compose exec -T drupal with-contenv bash -lc "for_all_sites update_settings_php" # Make sure the host user can read the settings.php files after they have been updated. sudo find ./codebase -type f -name "settings.php" -exec chown $(shell id -u):101 {} \; -# Updates configuration from environment variables. -# Allow all commands to fail as the user may not have all the modules like matomo, etc. .PHONY: update-config-from-environment +## Updates configuration from environment variables. +## Allow all commands to fail as the user may not have all the modules like matomo, etc. .SILENT: update-config-from-environment update-config-from-environment: - -docker-compose exec drupal with-contenv bash -lc "for_all_sites configure_islandora_module" - -docker-compose exec drupal with-contenv bash -lc "for_all_sites configure_matomo_module" - -docker-compose exec drupal with-contenv bash -lc "for_all_sites configure_openseadragon" - -docker-compose exec drupal with-contenv bash -lc "for_all_sites configure_islandora_default_module" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_islandora_module" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_jwt_module" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_islandora_default_module" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_search_api_solr_module" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_matomo_module" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_openseadragon" + -docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_islandora_default_module" -# Runs migrations of islandora .PHONY: run-islandora-migrations +## Runs migrations of islandora .SILENT: run-islandora-migrations run-islandora-migrations: - docker-compose exec drupal with-contenv bash -lc "for_all_sites import_islandora_migrations" + #docker-compose exec -T drupal with-contenv bash -lc "for_all_sites import_islandora_migrations" + # this line can be reverted when https://github.com/Islandora-Devops/isle-buildkit/blob/fae704f065435438828c568def2a0cc926cc4b6b/drupal/rootfs/etc/islandora/utilities.sh#L557 + # has been updated to match + docker-compose exec -T drupal with-contenv bash -lc 'drush -l $(SITE) migrate:import $(MIGRATE_IMPORT_USER_OPTION) islandora_defaults_tags,islandora_tags' -# Creates solr-cores according to the environment variables. .PHONY: solr-cores +## Creates solr-cores according to the environment variables. .SILENT: solr-cores solr-cores: - docker-compose exec drupal with-contenv bash -lc "for_all_sites create_solr_core_with_default_config" + docker-compose exec -T drupal with-contenv bash -lc "for_all_sites create_solr_core_with_default_config" -# Creates namespaces in Blazegraph according to the environment variables. .PHONY: namespaces +## Creates namespaces in Blazegraph according to the environment variables. .SILENT: namespaces namespaces: - docker-compose exec drupal with-contenv bash -lc "for_all_sites create_blazegraph_namespace_with_default_properties" + docker-compose exec -T drupal with-contenv bash -lc "for_all_sites create_blazegraph_namespace_with_default_properties" -# Reconstitute the site from environment variables. .PHONY: hydrate .SILENT: hydrate +## Reconstitute the site from environment variables. hydrate: update-settings-php update-config-from-environment solr-cores namespaces run-islandora-migrations - docker-compose exec drupal drush cr -y + docker-compose exec -T drupal drush cr -y # Created by the standard profile, need to be deleted to import a site that was # created with the standard profile. .PHONY: delete-shortcut-entities .SILENT: delete-shortcut-entities delete-shortcut-entities: - docker-compose exec drupal drush -l $(SITE) entity:delete shortcut_set + docker-compose exec -T drupal drush -l $(SITE) entity:delete shortcut_set -# Forces the site uuid to match that in the config_sync_directory so that +# Forces the site uuid to match that in the config_sync_directory so that # configuration can be imported. .PHONY: set-site-uuid .SILENT: set-site-uuid set-site-uuid: - docker-compose exec drupal with-contenv bash -lc "set_site_uuid" + docker-compose exec -T drupal with-contenv bash -lc "set_site_uuid" -# RemovesForces the site uuid to match that in the config_sync_directory so that +# RemovesForces the site uuid to match that in the config_sync_directory so that # configuration can be imported. .PHONY: remove_standard_profile_references_from_config .SILENT: remove_standard_profile_references_from_config remove_standard_profile_references_from_config: - docker-compose exec drupal with-contenv bash -lc "remove_standard_profile_references_from_config" + docker-compose exec -T drupal with-contenv bash -lc "remove_standard_profile_references_from_config" -# Exports the sites configuration. .PHONY: config-export .SILENT: config-export +## Exports the sites configuration. config-export: - docker-compose exec drupal drush -l $(SITE) config:export -y + docker-compose exec -T drupal drush -l $(SITE) config:export -y -# Import the sites configuration. -# N.B You may need to run this multiple times in succession due to errors in the configurations dependencies. .PHONY: config-import .SILENT: config-import +## Import the sites configuration. N.B You may need to run this multiple times in succession due to errors in the configurations dependencies. config-import: set-site-uuid delete-shortcut-entities - docker-compose exec drupal drush -l $(SITE) config:import -y + docker-compose exec -T drupal drush -l $(SITE) config:import -y -# Dump database. -database-dump: +drupal-database-dump: ifndef DEST $(error DEST is not set) endif -ifeq ($(wildcard $(CURDIR)/codebase),) - $(error codebase folder does not exists) -endif - docker-compose exec drupal drush -l $(SITE) sql:dump > /tmp/dump.sql + docker-compose exec -T drupal with-contenv bash -lc 'mysqldump -u $${DRUPAL_DEFAULT_DB_ROOT_USER} -p$${DRUPAL_DEFAULT_DB_ROOT_PASSWORD} -h $${DRUPAL_DEFAULT_DB_HOST} $${DRUPAL_DEFAULT_DB_NAME} > /tmp/dump.sql' docker cp $$(docker-compose ps -q drupal):/tmp/dump.sql $(DEST) # Import database. -database-import: $(SRC) +drupal-database-import: $(SRC) ifndef SRC $(error SRC is not set) endif -ifeq ($(wildcard $(CURDIR)/codebase),) - $(error codebase folder does not exists) -endif - docker cp $(SRC) $$(docker-compose ps -q drupal):/tmp/dump.sql + docker cp "$(SRC)" $$(docker-compose ps -q drupal):/tmp/dump.sql # Need to specify the root user to import the database otherwise it will fail due to permissions. - docker-compose exec drupal with-contenv bash -lc '`drush -l $(SITE) sql:connect --extra="-u $${DRUPAL_DEFAULT_DB_ROOT_USER} --password=$${DRUPAL_DEFAULT_DB_ROOT_PASSWORD}"` < /tmp/dump.sql' - -# Creates the codebase folder from a running islandora/demo image. -.PHONY: create-codebase-from-demo -.SILENT: create-codebase-from-demo -create-codebase-from-demo: -ifneq ($(wildcard $(CURDIR)/codebase),) - $(error codebase folder already exists) + docker-compose exec -T drupal with-contenv bash -lc 'chown root:root /tmp/dump.sql && mysql -u $${DRUPAL_DEFAULT_DB_ROOT_USER} -p$${DRUPAL_DEFAULT_DB_ROOT_PASSWORD} -h $${DRUPAL_DEFAULT_DB_HOST} $${DRUPAL_DEFAULT_DB_NAME} < /tmp/dump.sql' + +drupal-public-files-dump: +ifndef DEST + $(error DEST is not set) endif - # Create docker-compose.yml file for the demo environment. - $(MAKE) -B docker-compose.yml ENVIRONMENT=demo - # Ensure we have the latest images. - $(MAKE) pull - # Start the services - docker-compose up -d - # Give an extra few seconds for the containers to become responsive. - sleep 5 - # Wait for Drupal to become responsive (up to 5 minutes). - docker-compose exec drupal timeout 300 wait-for-open-port.sh localhost 80 - # Export the site configuration. - docker-compose exec drupal drush config:export - # Need `default` folder to be writeable to copy it down to host. - docker-compose exec drupal chmod 777 /var/www/drupal/web/sites/default - docker cp $$(docker-compose ps -q drupal):/var/www/drupal/ codebase - # Restore expected perms for `default`. - docker-compose exec drupal chmod 555 /var/www/drupal/web/sites/default - # Take down the services - docker-compose down -v - # Change ownership so the host user can work with the files. - sudo chown -R $(shell id -u):101 $(CURDIR)/codebase - # For newly added files/directories makesure they inherit the parent folders owner/group. - find $(CURDIR)/codebase -type d -exec chmod u+s,g+s {} \; - # Restore the docker-compose.yml file to what the user had before. - $(MAKE) -B docker-compose.yml + docker-compose exec -T drupal with-contenv bash -lc 'tar zcvf /tmp/public-files.tgz /var/www/drupal/web/sites/default/files' + docker cp $$(docker-compose ps -q drupal):/tmp/public-files.tgz $(DEST) -# Helper function to generate keys for the user to use in their docker-compose.env.yml -.PHONY: generate-jwt-keys -.SILENT: generate-jwt-keys -generate-jwt-keys: - docker run --rm -ti \ +drupal-public-files-import: $(SRC) +ifndef SRC + $(error SRC is not set) +endif + docker cp "$(SRC)" $$(docker-compose ps -q drupal):/tmp/public-files.tgz + docker-compose exec -T drupal with-contenv bash -lc 'tar zxvf /tmp/public-files.tgz -C /var/www/drupal/web/sites/default/files && chown -R nginx:nginx /var/www/drupal/web/sites/default/files && rm /tmp/public-files.tgz' + +# Composer Update +composer_update: + docker-compose exec -T drupal with-contenv bash -lc 'composer update' + +# Dump fcrepo. +fcrepo-export: +ifndef DEST + $(error DEST is not set) +endif + docker-compose exec -T fcrepo with-contenv bash -lc 'java -jar /opt/tomcat/fcrepo-import-export-1.0.1.jar --mode export -r http://$(DOMAIN):8081/fcrepo/rest -d /tmp/fcrepo-export -b -u $${FCREPO_TOMCAT_ADMIN_USER}:$${FCREPO_TOMCAT_ADMIN_PASSWORD}' + docker-compose exec -T fcrepo with-contenv bash -lc 'cd /tmp && tar zcvf fcrepo-export.tgz fcrepo-export' + docker cp $$(docker-compose ps -q fcrepo):/tmp/fcrepo-export.tgz $(DEST) + +# Import fcrepo. +fcrepo-import: $(SRC) +ifndef SRC + $(error SRC is not set) +endif + $(MAKE) -B docker-compose.yml DISABLE_SYN=true + docker-compose up -d fcrepo + docker cp "$(SRC)" $$(docker-compose ps -q fcrepo):/tmp/fcrepo-export.tgz + docker-compose exec -T fcrepo with-contenv bash -lc 'cd /tmp && tar zxvf fcrepo-export.tgz && chown -R tomcat:tomcat fcrepo-export && rm fcrepo-export.tgz' +ifeq ($(FEDORA_6), true) + docker-compose exec -T fcrepo with-contenv bash -lc 'java -jar fcrepo-upgrade-utils-6.0.0-beta-1.jar -i /tmp/fcrepo-export -o /data/home -s 5+ -t 6+ -u http://${DOMAIN}:8081/fcrepo/rest && chown -R tomcat:tomcat /data/home' +ifeq ($(FCREPO_DATABASE_SERVICE), postgresql) + $(error Postgresql not implemented yet in fcrepo-import) +else + docker-compose exec -T fcrepo with-contenv bash -lc 'mysql -u $${DB_ROOT_USER} -p$${DB_ROOT_PASSWORD} -h $${DB_MYSQL_HOST} -e "DROP DATABASE $${FCREPO_DB_NAME}"' +endif +else + docker-compose exec -T fcrepo with-contenv bash -lc 'java -jar /opt/tomcat/fcrepo-import-export-1.0.1.jar --mode import -r http://$(DOMAIN):8081/fcrepo/rest --map http://islandora.traefik.me:8081/fcrepo/rest,http://$(DOMAIN):8081/fcrepo/rest -d /tmp/fcrepo-export -b -u $${TOMCAT_ADMIN_NAME}:$${TOMCAT_ADMIN_PASSWORD}' +endif + $(MAKE) -B docker-compose.yml + docker-compose up -d fcrepo + +reindex-fcrepo-metadata: + # Re-index RDF in Fedora + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec non_fedora_files emit_file_event --configuration="queue=islandora-indexing-fcrepo-file-external&event=Update"' + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec all_taxonomy_terms emit_term_event --configuration="queue=islandora-indexing-fcrepo-content&event=Update"' + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec content emit_node_event --configuration="queue=islandora-indexing-fcrepo-content&event=Update"' + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec media emit_media_event --configuration="queue=islandora-indexing-fcrepo-media&event=Update"' + +reindex-solr: + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} search-api-reindex' + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} search-api-index' + +reindex-triplestore: + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec all_taxonomy_terms emit_term_event --configuration="queue=islandora-indexing-triplestore-index&event=Update"' + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec content emit_node_event --configuration="queue=islandora-indexing-triplestore-index&event=Update"' + docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec media emit_media_event --configuration="queue=islandora-indexing-triplestore-index&event=Update"' + +.PHONY: generate-secrets +## Helper to generate secrets & passwords, like so: make generate-secrets +.SILENT: generate-secrets +generate-secrets: +ifeq ($(USE_SECRETS), false) + docker run --rm -t \ + -v "$(CURDIR)/secrets":/secrets \ + -v "$(CURDIR)/build/scripts/generate-secrets.sh":/generate-secrets.sh \ + -w / \ --entrypoint bash \ - $(REPOSITORY)/drupal:$(TAG) -c \ - "openssl genrsa -out /tmp/private.key 2048 &> /dev/null; \ - openssl rsa -pubout -in /tmp/private.key -out /tmp/public.key &> /dev/null; \ - echo $$'\nPrivate Key:\n'; \ - cat /tmp/private.key; \ - echo $$'\nPublic Key:\n'; \ - cat /tmp/public.key; \ - echo $$'\nCopy and paste these keys into your docker-compose.env.yml file where appropriate.'" - -# Helper to generate Matomo password, like so: -# make generate-matomo-password MATOMO_USER_PASS=my_new_password -.PHONY: generate-matomo-password -.SILENT: generate-matomo-password -generate-matomo-password: -ifndef MATOMO_USER_PASS - $(error MATOMO_USER_PASS is not set) + $(REPOSITORY)/drupal:$(TAG) -c "/generate-secrets.sh && chown -R `id -u`:`id -g` /secrets" +else + @echo "'Uses Secrets' is set to 'true'." + $(MAKE) secrets_warning endif - docker run --rm -ti \ - --entrypoint php \ - $(REPOSITORY)/drupal:$(TAG) -r \ - 'echo password_hash(md5("$(MATOMO_USER_PASS)"), PASSWORD_DEFAULT) . "\n";' -# Helper function to generate keys for the user to use in their docker-compose.env.yml .PHONY: download-default-certs +## Helper function to generate keys for the user to use in their docker-compose.env.yml .SILENT: download-default-certs download-default-certs: mkdir -p certs @@ -286,36 +334,234 @@ download-default-certs: curl http://traefik.me/privkey.pem -o certs/privkey.pem; \ fi -.PHONY: dev -.SILENT: dev -dev: - $(MAKE) download-default-certs - $(MAKE) create-codebase-from-demo - if grep -q DRUPAL_DEFAULT_CONFIGDIR docker-compose.env.yml; then \ - sed -i 's/DRUPAL_DEFAULT_CONFIGDIR:.*/DRUPAL_DEFAULT_CONFIGDIR: \/var\/www\/drupal\/config\/sync/g' docker-compose.env.yml; \ - else \ - sed -i '/DRUPAL_DEFAULT_SALT/a \ \ \ \ \ \ DRUPAL_DEFAULT_CONFIGDIR: \/var\/www\/drupal\/config\/sync' docker-compose.env.yml; \ - fi - if grep -q DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG docker-compose.env.yml; then \ - sed -i 's/DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG:.*/DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG: "true"/g' docker-compose.env.yml; \ - else \ - sed -i '/DRUPAL_DEFAULT_SALT/a \ \ \ \ \ \ DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG: "true"' docker-compose.env.yml; \ - fi - if grep -q DRUPAL_DEFAULT_PROFILE docker-compose.env.yml; then \ - sed -i 's/DRUPAL_DEFAULT_PROFILE:.*/DRUPAL_DEFAULT_PROFILE: minimal/g' docker-compose.env.yml; \ - else \ - sed -i '/DRUPAL_DEFAULT_SALT/a \ \ \ \ \ \ DRUPAL_DEFAULT_PROFILE: minimal' docker-compose.env.yml; \ - fi +.PHONY: demo +.SILENT: demo +## Make a local site from the install-profile and TODO then add demo content +demo: generate-secrets + $(MAKE) local + $(MAKE) demo_content + $(MAKE) login + +.PHONY: local +#.SILENT: local +## Make a local site with codebase directory bind mounted, modeled after sandbox.islandora.ca +local: QUOTED_CURDIR = "$(CURDIR)" +local: generate-secrets + $(MAKE) download-default-certs ENVIRONMENT=local $(MAKE) -B docker-compose.yml ENVIRONMENT=local - docker-compose up -d - $(MAKE) remove_standard_profile_references_from_config - $(MAKE) install ENVIRONMENT=local + $(MAKE) pull ENVIRONMENT=local + mkdir -p $(CURDIR)/codebase + if [ -z "$$(ls -A $(QUOTED_CURDIR)/codebase)" ]; then \ + docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone -b main https://github.com/islandora-devops/islandora-sandbox /tmp/codebase; mv /tmp/codebase/* /home/root;'; \ + fi + $(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=local + docker-compose up -d --remove-orphans + docker-compose exec -T drupal with-contenv bash -lc 'composer install; chown -R nginx:nginx .' + $(MAKE) remove_standard_profile_references_from_config drupal-database update-settings-php ENVIRONMENT=local + docker-compose exec -T drupal with-contenv bash -lc "drush si -y islandora_install_profile_demo --account-pass $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)" + $(MAKE) delete-shortcut-entities && docker-compose exec -T drupal with-contenv bash -lc "drush pm:un -y shortcut" + docker-compose exec -T drupal with-contenv bash -lc "drush en -y migrate_tools" $(MAKE) hydrate ENVIRONMENT=local + -docker-compose exec -T drupal with-contenv bash -lc 'mkdir -p /var/www/drupal/config/sync && chmod -R 775 /var/www/drupal/config/sync' + #docker-compose exec -T drupal with-contenv bash -lc 'chown -R `id -u`:nginx /var/www/drupal' + #docker-compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags' + curl -k -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@build/demo-data/homepage.json" https://${DOMAIN}/node?_format=json + curl -k -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@build/demo-data/browse-collections.json" https://${DOMAIN}/node?_format=json + $(MAKE) login + +.PHONY: demo_content +#.SILENT: demo_content +## Helper function for demo sites: do a workbench import of sample objects +demo_content: + # fetch repo that has csv and binaries to data/samples + # if prod do this by default + # if [ -d "islandora_workbench" ]; then rm -rf islandora_workbench; fi + [ -d "islandora_workbench" ] || (git clone -b new_staging --single-branch https://github.com/DonRichards/islandora_workbench) + $(SED_DASH_I) 's/^nopassword.*/password\: $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) /g' islandora_workbench/demoBDcreate* + $(SED_DASH_I) 's/http:/https:/g' islandora_workbench/demoBDcreate* + $(SED_DASH_I) 's/author_email\="mjordan@sfu"\,$$/author_email="mjordan@sfu", packages=["i7Import", "i8demo_BD", "input_data"],/g' islandora_workbench/setup.py + cd islandora_workbench && docker build -t workbench-docker . + cd islandora_workbench && docker run -it --rm --network="host" -v $(shell pwd)/islandora_workbench:/workbench --name my-running-workbench workbench-docker bash -lc "(cd /workbench && python setup.py install 2>&1 && ./workbench --config demoBDcreate_all_localhost.yml)" + $(MAKE) reindex-solr -# Destroys everything beware! .PHONY: clean .SILENT: clean +## Destroys everything beware! clean: + echo "**DANGER** About to rm your SERVER data subdirs, your docker volumes, codebase, islandora_workbench, certs, secrets, and all untracked/ignored files (including .env)." + $(MAKE) confirm -docker-compose down -v - sudo rm -fr codebase certs + sudo rm -fr codebase islandora_workbench certs secrets/live/* git clean -xffd . + +.PHONY: up +.SILENT: up +## Brings up the containers or builds demo if no containers were found. +up: + test -f docker-compose.yml && docker-compose up -d --remove-orphans || $(MAKE) demo + @echo "\n Sleeping for 10 seconds to wait for Drupal to finish building.\n" + sleep 10 + docker-compose exec -T drupal with-contenv bash -lc "for_all_sites update_settings_php" + $(MAKE) secrets_warning + +.PHONY: down +.SILENT: down +## Brings down the containers. Same as docker-compose down --remove-orphans +down: + -docker-compose down --remove-orphans + +.PHONY: login +.SILENT: login +## Runs "drush uli" to provide a direct login link for user 1 +login: + echo "\n\n=========== LOGIN ===========" + docker-compose exec -T drupal with-contenv bash -lc "drush uli --uri=$(DOMAIN)" + echo "=============================\n" + +.PHONY: env +.SILENT: env +## Pull in changes to the .env file. +env: + if [ -f .env ]; then \ + $(MAKE) down ; \ + $(MAKE) -B docker-compose.yml ; \ + $(MAKE) pull ; \ + $(MAKE) up ; \ + echo -e '\n\n${BLUE}Fixing the error message: ${RESET} ${RED}In Filesystem.php line 203${RESET}\n\n' ; \ + docker-compose exec -T drupal with-contenv bash -lc "cp /var/www/drupal/web/sites/default/settings.php /var/www/drupal/web/sites/default/settings.php.bak" ; \ + docker-compose exec -T drupal with-contenv bash -lc "cp /var/www/drupal/web/sites/default/default.settings.php /var/www/drupal/web/sites/default/settings.php" ; \ + docker-compose exec -T drupal with-contenv bash -lc "chown nginx:nginx /var/www/drupal/web/sites/default/settings.php" ; \ + docker-compose exec -T drupal with-contenv bash -lc "chmod 644 /var/www/drupal/web/sites/default/settings.php" ; \ + $(MAKE) update-settings-php ; \ + fi + if [ ! -f .env ]; then \ + echo "No .env file found." ; \ + fi + +.phony: confirm +confirm: + @echo -n "Are you sure you want to continue and drop your data? [y/N] " && read ans && [ $${ans:-N} = y ] + +RESET=$(shell tput sgr0) +RED=$(shell tput setaf 9) +BLUE=$(shell tput setaf 6) +TARGET_MAX_CHAR_NUM=20 + +.PHONY: help +.SILENT: help +help: + @echo '' + @echo 'Usage:' + @echo ' ${RED}make${RESET} ${BLUE}${RESET}' + @echo '' + @echo 'Functions to build:' + # @grep '^.PHONY: .* #' Makefile | sed 's/\.PHONY: \(.*\) # \(.*\)/\1 \2/' + @awk '/^[a-zA-Z\-\_0-9]+:/ { \ + helpMessage = match(lastLine, /^## (.*)/); \ + if (helpMessage) { \ + helpCommand = $$1; sub(/:$$/, "", helpCommand); \ + helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \ + if (helpCommand == "up" || helpCommand == "local" || helpCommand == "demo") { \ + printf " ${RED}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${BLUE}%s${RESET}\n", helpCommand, helpMessage; \ + } \ + } \ + } \ + {lastLine = $$0}' $(MAKEFILE_LIST) + @echo '' + @echo 'Other functions:' + @awk '/^[a-zA-Z\-\_0-9]+:/ { \ + helpMessage = match(lastLine, /^## (.*)/); \ + if (helpMessage) { \ + helpCommand = $$1; sub(/:$$/, "", helpCommand); \ + helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \ + if (helpCommand != "up" && helpCommand != "local" && helpCommand != "demo") { \ + printf " ${RED}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${BLUE}%s${RESET}\n", helpCommand, helpMessage ; \ + } \ + } \ + } \ + {lastLine = $$0}' $(MAKEFILE_LIST) + @echo '' + +.PHONY: secrets_warning +.SILENT: secrets_warning +## Check to see if the secrets directory contains default secrets. +secrets_warning: + @echo 'Starting build/scripts/check-secrets.sh' + @bash build/scripts/check-secrets.sh || (echo "check-secrets exited $$?"; exit 1) + +IS_DRUPAL_PSSWD_FILE_READABLE := $(shell test -r secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD -a -w secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD && echo 1 || echo 0) +CMD := $(shell [ $(IS_DRUPAL_PSSWD_FILE_READABLE) -eq 1 ] && echo 'tee' || echo 'sudo -k tee') + +.PHONY: set_admin_password +.SILENT: set_admin_password +## Sets the admin password and accomodates for permissions restrictions to the secrets directory. Only runs sudo when needed. +set_admin_password: + @$(eval PASSWORD ?= $(shell bash -c 'read -s -p "New Password: " pwd; echo $$pwd')) + @echo "\n\nSetting admin password now" + docker-compose exec -T drupal with-contenv bash -lc 'drush user:password admin "$(PASSWORD)"' + echo "$(PASSWORD)" | $(CMD) secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD >> /dev/null + @echo "\ndone." + +# Hot fix section. These are not meant to be run normally but are meant to be run when needed. + +LATEST_VERSION := $(shell curl -s https://api.github.com/repos/desandro/masonry/releases/latest | grep '\"tag_name\":' | sed -E 's/.*\"([^\"]+)\".*/\1/') + +.PHONY: fix_masonry +.SILENT: fix_masonry +## Fix missing masonry library. +fix_masonry: + @echo "Latest version of masonry library is ${LATEST_VERSION}" + docker-compose exec drupal bash -lc "[ -d '/var/www/drupal/web/libraries' ] && exit ; mkdir -p /var/www/drupal/web/libraries ; chmod 755 /var/www/drupal/web/libraries ; chown 1000:nginx /var/www/drupal/web/libraries" + docker-compose exec drupal bash -lc "cd /var/www/drupal/web/libraries/ ; [ ! -d '/var/www/drupal/web/libraries/masonry' ] && git clone --quiet --branch ${LATEST_VERSION} https://github.com/desandro/masonry.git || echo Ready" + docker-compose exec drupal bash -lc "cd /var/www/drupal/web/libraries/ ; [ -d '/var/www/drupal/web/libraries/masonry' ] && chmod -R 755 /var/www/drupal/web/libraries/masonry ; chown -R 1000:nginx /var/www/drupal/web/libraries/masonry" + +.PHONY: fix_views +.SILENT: fix_views +## This fixes a know issues with views when using the make local build. The error must be triggered before this will work. +fix_views: + docker cp scripts/patch_views.sh $$(docker ps --format "{{.Names}}" | grep drupal):/var/www/drupal/patch_views.sh + docker-compose exec -T drupal with-contenv bash -lc "bash /var/www/drupal/patch_views.sh ; rm /var/www/drupal/patch_views.sh ; drush cr" + +.PHONY: starter +## Make a local site with codebase directory bind mounted, using starter site. +starter: QUOTED_CURDIR = "$(CURDIR)" +starter: generate-secrets + $(MAKE) starter-init ENVIRONMENT=starter + if [ -z "$$(ls -A $(QUOTED_CURDIR)/codebase)" ]; then \ + docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'composer create-project islandora/islandora-starter-site:dev-main /tmp/codebase; mv /tmp/codebase/* /home/root;'; \ + fi + $(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=starter + docker-compose up -d --remove-orphans + $(MAKE) starter-finalize ENVIRONMENT=starter + +.PHONY: starter_dev +## Make a local site with codebase directory bind mounted, using cloned starter site. +starter_dev: QUOTED_CURDIR = "$(CURDIR)" +starter_dev: generate-secrets + $(MAKE) starter-init ENVIRONMENT=starter_dev + if [ -z "$$(ls -A $(QUOTED_CURDIR)/codebase)" ]; then \ + docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone -b main https://github.com/Islandora-Devops/islandora-starter-site /home/root;'; \ + fi + $(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=starter_dev + docker-compose up -d --remove-orphans + docker-compose exec -T drupal with-contenv bash -lc 'composer install' + $(MAKE) starter-finalize ENVIRONMENT=starter_dev + +.PHONY: starter-init +starter-init: generate-secrets + $(MAKE) download-default-certs + $(MAKE) -B docker-compose.yml + $(MAKE) pull + mkdir -p $(CURDIR)/codebase + +.PHONY: starter-finalize +starter-finalize: + docker-compose exec -T drupal with-contenv bash -lc 'chown -R nginx:nginx .' + $(MAKE) drupal-database update-settings-php + docker-compose exec -T drupal with-contenv bash -lc "drush si -y --existing-config minimal --account-pass $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)" + docker-compose exec -T drupal with-contenv bash -lc "drush -l $(SITE) user:role:add fedoraadmin admin" + MIGRATE_IMPORT_USER_OPTION=--userid=1 $(MAKE) hydrate + docker-compose exec -T drupal with-contenv bash -lc 'drush -l $(SITE) migrate:import --userid=1 islandora_fits_tags' + #docker-compose exec -T drupal with-contenv bash -lc 'chown -R `id -u`:nginx /var/www/drupal' + #docker-compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags' + $(MAKE) login \ No newline at end of file diff --git a/README.md b/README.md index fd5f44904..f4487c53b 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,19 @@ -# ISLE: Islandora Enterprise 8 Prototype +# ISLE: Islandora Enterprise 2 [![LICENSE](https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](./LICENSE) - [Introduction](#introduction) - [Requirements](#requirements) -- [Installation](#installation) - - [Configuring the Environment](#configuring-the-environment) - - [Changing the host name](#changing-the-host-name) - - [Using an IP address](#using-an-ip-address) - - [Applying changes](#applying-changes) -- [Demo Environment](#demo-environment) -- [Local Environment](#local-environment) - - [Create Local Environment from islandora/demo Image](#create-local-environment-from-islandorademo-image) - - [From existing Configuration](#from-existing-configuration) - - [Manually](#manually) - - [Create Local Environment from Existing Site](#create-local-environment-from-existing-site) - - [Create Local Environment from Scratch](#create-local-environment-from-scratch) +- [Getting Started](#getting-started) +- [Local Development](#local-development) - [Custom Environment](#custom-environment) +- [Secrets](#secrets) - [Services](#services) + - [Code Server](#code-server) - [Watchtower](#watchtower) - [Traefik](#traefik) - [ETCD](#etcd) - [Troubleshooting/Issues](#troubleshootingissues) -- [FAQ](#faq) - - [Question: When doing an `drush config:import -y` I get one of the following errors](#question-when-doing-an-drush-configimport--y-i-get-one-of-the-following-errors) - - [Question: When doing an `drush sql:dump` I get one of the following errors](#question-when-doing-an-drush-sqldump-i-get-one-of-the-following-errors) - [Development](#development) - [Maintainers/Sponsors](#maintainerssponsors) - [Architecture Team](#architecture-team) @@ -33,357 +22,292 @@ ## Introduction -[Docker Compose] project facilitating creation and management of Islandora 8 -Infrastructure under [Docker] using [Docker Compose]. +[Docker Compose] project for creating and managing an Islandora 8 instance +using [Docker] containers from [Docker Hub](https://hub.docker.com/u/islandora) +that were created by [isle-buildkit](https://github.com/Islandora-Devops/isle-buildkit). -This is a prototype of the `docker-compose` file, Docker service and image -configuration structure for the ISLE Phase III - ISLE / Islandora 8 Prototype -(isle-dc) project. +In a nutshell, `isle-dc` generates a docker-compose.yml file for you based on configuration +that you supply in a `.env` file. And there are three use cases we're trying to accomplish: -The workflow for this repository centers around using the provided -[Makefile](./Makefile) to generate an appropriate `docker-compose.yml` file. +- **demo** *(Example site for kicking the tires and looking at Islandora)* +- **local** *(Local development using composer/drush in the codebase folder)* +- **custom** *(A custom Dockerfile to deploy created from local)* -There are **three** `ENVIRONMENT`s or ways of development that this repository -supports: +Additionally, there's a couple other targets derived from `local` which make use of [the `islandora/islandora-starter-site` project](https://github.com/Islandora/islandora-starter-site): -- **demo** *(Example site for testing the images)* -- **local** *(Local development using composer/drush in the codebase folder)* -- **custom** *(Use a custom built image or generate one from the codebase folder)* +- **starter**: Uses `composer create-project` to initialize the site, for general use; and, +- **starter_dev**: Creates a clone of the starter site project, intended for development of the "starter site" proper; however, given a number of different items are configured during provisioning, `starter_dev` may be of limited utility as config exports will be dirtied during provisioning (ideally, these bits that vary could be reworked to use [Drupal's "state API"](https://www.drupal.org/docs/8/api/state-api/overview) instead, or perhaps avoiding reworking of the modules by using [Drupal's configuration override system](https://www.drupal.org/docs/drupal-apis/configuration-api/configuration-override-system)). -To quickly get started, we recommend running the [demo](#-demo) environment -first after you have completed the [Installation](#-installation). +On top of that, there's a lot of useful commands for managing an Islandora instance, such +as database import/export and reindexing. ## Requirements -- Composer 1.10+ -- Desktop / laptop / VM (*Must be able to support running GNU Make*) -- Docker-CE 19.x+ (If using Docker Desktop for Windows, any stable release - *after* 2.2.0.4, or use a 2.2.0.4 with a [patch][Docker for Windows Patch] due - to a [bug][Docker for Windows Bug]) +- Desktop / laptop / VM (*Docker must have sufficient resources to run GNU Make*) +- Docker-CE 19.x+ - Docker-compose version 1.25.x+ -- Drush 9.0+ - Git 2.0+ - GNU Make 4.0+ -- PHP 7.2+ (*Also requires the same ext packages you intend to use in your site.*) - -## Installation - -### Configuring the Environment - -To run the containers you must first generate a `docker-compose.yml` file. It is -the only orchestration mechanism provided to launch all the containers, and have -them work as a whole. +- At least 8GB of RAM (ideally 16GB) -To get started generate the defaults with the following command: +before running any of the make commands below. -```bash -make -``` - -This will create the following files which you can **customize**: -| File | Purpose | -| :----------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `.env` | Responsible for setting variables used in `docker-compose.*.yml` files
Determines which `docker-compose.*.yml` are included in `docker-compose.yml` | -| `docker-compose.env.yml` | Allows the user to set environment settings inside of containers and override any services configuration | +See release notes at https://docs.docker.com/compose/cli-command/. -At a minimum, you'll want to consider setting `ENVIRONMENT` in the `.env` file to either `demo`, `local`, or `custom`. The default is `demo`. -#### Changing the host name +## Getting Started -By default, the domain `traefik.me` is used, which resolves to `localhost`, but allows us to treat things as if there were a fully qualified domain name. Namely, we can have https in development and staging scenarios, even when all you have is an IP address. - -However, if you are deploying somewhere other than `localhost` and you own a domain, you can change it by setting `DRUPAL_SITE_HOST` in the .env file. That is, -for `example.org`: +To get started with a **demo** environment, run: ```bash -DRUPAL_SITE_HOST=example.org +make demo ``` -#### Using an IP address +⚠️ If prompted during `make up\demo\local\clean` for password, use your computer's password. The build process may need elevated privileges to write or remove files. For other password information see [Secrets](#secrets) -If you have an IP address but no domain, you can set the value to `X-X-X-X.traefik.me`, where X-X-X-X is your IP address, but with hyphens -instead of dots. For example, if your IP address is `123.45.67.89`: +This will pull down images from Dockerhub and generate -```bash -DRUPAL_SITE_HOST=123-45-67-89.traefik.me -``` +| File | Purpose | +| :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.env` | A configuration file that is yours to customize. This file controls how the docker-compose.yml file gets generated to meet your use case.
It also allows you to set variables that make their way into the final `docker-compose.yml` file, such as your site's domain. | +| `docker-compose.yml` | A ready to run `docker-compose.yml` file based on your `.env` file. This file is considered disposable. When you change your `.env` file, you will generate a new one. | -There are also a handful of variables in `docker-compose.env.yml` you'll want to adjust if using an IP address with traefik.me. For each of these, -change the dot between COMPOSE_PROJECT_NAME and DRUPAL_SITE_HOST to a hyphen (i.e. ${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} -becomes ${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}). If you have any doubts about what you're doing, just copy/paste these values -directly into place in your `docker-compose.env.yml` file. +Your new Islandora instance will be available at +[https://islandora.traefik.me](https://islandora.traefik.me). Don't let the +funny URL fool you, it's a dummy domain that resolves to `127.0.0.1`. -| Variable | Value | -| :---------------------------- | :--------------------------------------------------------------------------------------------------- | -| DRUPAL_DEFAULT_CANTALOUPE_URL | | -| DRUPAL_DEFAULT_DB_HOST | ${DRUPAL_DATABASE_SERVICE}-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} | -| DRUPAL_DEFAULT_FCREPO_HOST | fcrepo-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} | -| DRUPAL_DEFAULT_MATOMO_URL | | -| DRUPAL_DEFAULT_SITE_URL | | -| MATOMO_SITE_HOST | islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} | +If you do not have [secrets enabled](#secrets), you can log into Drupal as +`admin` using the default password: `password`. Otherwise you can find the +password in the file +[./secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD](./secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD). -When using an IP address, your site will be available at https://islandora-isle-dc-X-X-X-X.traefik.me/, where X-X-X-X is your IP address. For example, https://islandora-isle-dc-123-45-67-89.traefik.me/ +Enjoy your Islandora instance! Check out the [Islandora documentation](https://islandora.github.io/documentation) to see all +the things you can do. If you want to poke around, here's all the services that are available to visit: -### Applying changes +| Service | URL | Exposed by default | +| :---------- | :--------------------------------------------------------------------------------------------- | :------------------ | +| Drupal | [https://islandora.traefik.me](https://islandora.traefik.me) | Yes | +| Traefik | [https://islandora.traefik.me:8080](https://islandora.traefik.me:8080) | No | +| Fedora | [https://islandora.traefik.me:8081/fcrepo/rest](https://islandora.traefik.me:8081/fcrepo/rest) | Yes | +| Blazegraph | [https://islandora.traefik.me:8082/bigdata](https://islandora.traefik.me:8082/bigdata) | No | +| Activemq | [http://islandora.traefik.me:8161](http://islandora.traefik.me:8161) | No | +| Solr | [http://islandora.traefik.me:8983](http://islandora.traefik.me:8983) | No | +| Cantaloupe | [https://islandora.traefik.me/cantaloupe](https://islandora.traefik.me/cantaloupe) | Yes | +| Matomo | [https://islandora.traefik.me/matomo/](https://islandora.traefik.me/matomo/) | Yes | +| Code Server | [https://islandora.traefik.me:8443/](https://islandora.traefik.me:8443/) | No | -Once you are happy with your changes to the above files you can regenerate the -`docker-compose.yml`, and pull the required images using the -[Makefile](./Makefile) like so: +> **Exposed**: the act of allowing the containerized application's ports to be accessible to the host machine (or public). In most cases this makes the specified URL available for the browser. -```bash -make +To change a service exposed value edit the *.env* file. The values will start with "EXPOSE_". Make changes then rebuild the docker-compose file and then run the up command (even if it's already running) using the following commands. +```shell +make -B docker-compose.yml +make up ``` -After this point you can just interact with the `docker-compose.yml` file like -you would normally. +When you're done with your demo environment, shut it down by running ```bash -docker-compose up -d +docker-compose down ``` -With [Docker Compose] there are many features such as displaying logs among -other things for which you can find detailed descriptions in the -[Docker Composer CLI Documentation](https://docs.docker.com/compose/reference/overview/) - -## Demo Environment - -To quickly get started, we recommend running the [demo](#-demo) environment first. +This will keep your data around until the next time you start your instance. If you want to completely destroy the repository and +all ingested data, use -This is the default environment if you do a clean checkout of this repository and run the following: - -```bash -make -docker-compose up -d ``` +docker-compose down -v -This environment is just meant as a show case of the basic `islandora` site. - -You should be able to reach it in your browser at `islandora-isle-dc.traefik.me` -if you followed the instructions under the [Installation](#-installation) -section. +# OR a better option because it removes these directories (codebase/ certs/ secrets/live/) and +# resets all files back to their original states. This takes it back to a clean slate. -## Local Environment +make clean -This environment is intended for local development. Users will create a `Drupal` -site in the folder [codebase](./codebase), which gets bind mounted into the -`drupal` service container. Allowing developers to use `composer` / `drush` -locally to work out of the [codebase](./codebase) folder. - -There are a three ways in which you can create a local environment. - -- From the `islandora/demo` image. -- From an existing site. -- From scratch. +``` -**N.B:** Before attempting any of these methods make sure you have set `ENVIRONMENT` to -`local` in your `.env` file. +## Local Development -After you have setup your local site you can then work directly out of the -`codebase` folder, as if you had installed `Drupal` locally. +When developing locally, your Drupal site resides in the `codebase` folder and is bind-mounted into your +Drupal container. This lets you update code using the IDE of your choice on your host machine, and the +changes are automatically reflected on the Drupal container. Simply place any exported Drupal site as +the `codebase` folder in `isle-dc` and you're good to go. -### Create Local Environment from islandora/demo Image +If you don't provide a codebase, you'll be given a basic setup from vanilla Drupal 9 instance with the Islandora module +installed and the bare minimum configured to run. This is useful if you want to build your repository +from scratch and avoid `islandora_defaults`. -The following [Makefile](./Makefile) command is provided as method to quickly -get a site up using the `islandora/demo` image as base. +If you've included configuration in your exported site using `drush config:export` or run `make config-export`, then you'll need +to set two values in your .env file: -```bash -make create-codebase-from-demo ``` - -It will take a few minutes to spin up the demo instance, export its -configuration and copy the site into the [codebase](./codebase) folder. - -Once this is done you can bring up your local site using `docker-compose`. - -```bash -docker-compose up -d +INSTALL_EXISTING_CONFIG=true +DRUPAL_INSTALL_PROFILE=minimal ``` -At this point the site will not be installed. There are **two** was to do an -installation, from an existing configuration or in a stepwise fashion. - -**After** either of your chosen methods you will still need to update services -like `solr` and `blazegraph`, etc, these commands are combined into a single -target for convenience. +In either case, run one of these commands to make a local environment. ```bash -make hydrate +make local ``` -#### From existing Configuration +The former will create a starter site modeled off of https://sandbox.islandora.ca. -To be able to install from an existing configuration you must change the -following environment variables in `docker-compose.env.yml`: +If you already have a Drupal site but don't know how to export it, +log into your server, navigate to the Drupal root, and run the following commands: -| Environment Variable | Value | -| :------------------------------------- | :-------------------------- | -| DRUPAL_DEFAULT_CONFIGDIR | /var/www/drupal/config/sync | -| DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG | "true" | -| DRUPAL_DEFAULT_PROFILE | minimal | +- `make config-export` +- `git init` +- `git add -A .` +- `git commit -m "First export of site"` -Regenerate your `docker-compose.yml` file, and restart the container. +Then you can `git push` your site to Github and `git clone` it down whenever you want. -```bash -make docker-compose.yml -docker-compose up -d -``` - -You also need to change the site configuration to use the minimal profile. - -```bash -make remove_standard_profile_references_from_config -``` +## Custom Environment -At this point you should be able to perform the installation +This environment is used to run your custom `drupal` image which can be produced +outside of this repository. You can specify the image in your `.env` file using +the settings `PROJECT_DRUPAL_DOCKERFILE` if you want to build it in the context +of this repository. You can also set the memory limits for each containers here as well. -```bash -make install -``` +For convenience a `sample.Dockerfile` is provided from which you can generate a +custom image from the [codebase](./codebase) folder. For example if you followed +the guide above to create the codebase folder from the `islandora/demo` image. -Finally configure the rest of the site which depends on environment variables. +And then run it by changing `ENVIRONMENT` to be `custom` and regenerating the +`docker-compose.yml` file and building the image. ```bash -make hydrate +make docker-compose.yml +make build ``` -**N.B.:** There is a -[bug](https://www.drupal.org/project/drupal/issues/2914213) which affects -`islandora_fits`. For now you must manually set a value. Visit - and set the value -`URL` to . - -#### Manually - - you can install it by running the -following [Makefile](./Makefile) command: +At this point you could run it using `docker-compose`: ```bash -make install +make up +# Or in some situations you could run this instead. +docker-compose up -d ``` -That will create the required database and install a bare bones site. - -Now that the `drupal` service is running you can update the `settings.php` with -the appropriate settings from the environment variables defined in -`docker-compose.env.yml`, by running the following [Makefile](./Makefile) -command: +To specify an image created outside of this repository, you can add the +following to `docker-compose.env.yml`: -```bash -make update-settings-php +```yaml +drupal: + image: YOUR_CUSTOM_IMAGE ``` -At this point the site is still bare-bones as we have not imported the site -configuration. Or you can manually setup the site as you see fit. To import an -existing configuration you can use the following command. +## Shutting down and bring back up +To run a non-destructive shutdown and bring it back up without having to know the docker commands needed. This keeps all of the commands for basic operations within the make commands. +```shell +# Shut down isle-dc with losing work. +make down -```bash -make config-import -``` +# Bring isle-dc back up from where it left off +make up -*N.B:* You'll have to run this twice, due to bugs in the dependencies of configurations! +# If make hasn't been run this will run make demo -Finally configure the rest of the site which depends on environment variables. - -```bash -make hydrate ``` -**N.B.:** There is a -[bug](https://www.drupal.org/project/drupal/issues/2914213) which affects -`islandora_fits`. For now you must manually set a value. Visit - and set the value -`URL` to . +## Secrets -### Create Local Environment from Existing Site +When running Islandora in the wild, you'll want to use secrets to store sensitive +information such as credentials. Secrets are communicated from the docker host +to the individual containers over an encrypted channel, making it much safer +to run in production. -Copy or clone your existing site into the [codebase](./codebase) folder. Start the system with `docker-compose up -d`, then run composer install via `docker-compose exec drupal with-contenv bash -lc 'COMPOSER_MEMORY_LIMIT=-1 composer install'` and make sure your database and user are created with `make databases`, and that your settings.php file is correct with `make update-settings-php`. +Some `confd` backends, such as `etcd`, can be used to serve secrets directly. +Simply expose `etcd` over `https` and nothing else needs to be done. But for +other backends, particularly environment variables, you must mount the secrets +into containers as files using docker-compose. During startup, the files' +contents are read into the container environment and made available to `confd`. -Then you have a number of options you can: +To enable using secrets prior to running the `make` commands, copy sample.env +to .env. Set `USE_SECRETS=true` in your .env file. Make a copy of the files in +/secrets/template/ to /secrets/live/. -- Follow the same installation procedure as the demo [from an existing configuration](#from-existing-configuration) -- Follow the same installation procedure as the demo [manually](#manually) +To enable using secrets after run `make local` or `make up`, set +`USE_SECRETS=true` in your .env file. When you run `make docker-compose.yml`, a +large block of `secrets` will be added at the top of your `docker-compose.yml` +file. -Or you can import an existing database if you have it. - -```bash -make database-import SRC=/tmp/dump.sql +```yml +secrets: + ACTIVEMQ_PASSWORD: + file: "./secrets/live/ACTIVEMQ_PASSWORD" + ACTIVEMQ_WEB_ADMIN_PASSWORD: + file: "./secrets/live/ACTIVEMQ_WEB_ADMIN_PASSWORD" + ... ``` -Finally configure the rest of the site which depends on environment variables. +Each secret references a file in the `secrets/live` directory. These files are +generated by `make`. Each secrets file is named the exact same as the +environment variable it intends to replace. The contents of each file will be +used as the value for the secret. -```bash -make hydrate +To automatically run secret generator without prompting (for creating a CICD/sandbox process) use: +```shell +bash scripts/check-secrets.sh yes ``` -### Create Local Environment from Scratch - -You can create a composer project for your drupal site from scratch. - -Some popular examples: +### Quick Drupal "admin" password reset +Run `make set_admin_password` and it will prompt the user to enter in a new password. Enter it in and the password for the "admin" user will be set to the new password. +```shell +$ make set_admin_password +Password: *** +Setting admin password now + [success] Changed password for admin. -- drupal/recommended-project -- drupal-composer/drupal-project:8.x-dev -- islandora/drupal-project:8.8.1 -- born-digital/drupal-project:dev-isle8-dev - -```bash -mkdir ./codebase -cd ./codebase -composer create-project --ignore-platform-reqs --no-interaction --no-install drupal/recommended-project . -composer require -- drush/drush -composer install -make -docker-compose up -d -make install ``` -At this point you should have a functioning Drupal site that you can customize -as you see fit using `composer` / `drush` commands in the codebase folder. +## Services -## Custom Environment +Islandora is composed of many different services, this project has split these +services up such that they have their own +[Container](https://www.docker.com/resources/what-container). -This environment is used to run your custom `drupal` image which can be produced -outside of this repository. You can specify the image in your `.env` file using -the settings `PROJECT_DRUPAL_DOCKERFILE` if you want to build it in the context -of this repository. +For in-depth documentation of the various `islandora` images see the +[isle-buildkit](https://github.com/Islandora-Devops/isle-buildkit) repository. -For convenience a `sample.Dockerfile` is provided from which you can generate a -custom image from the [codebase](./codebase) folder. For example if you followed -the guide above to create the codebase folder from the `islandora/demo` image. +Other services will be documented below: -And then run it by changing `ENVIRONMENT` to be `custom` and regenerating the -`docker-compose.yml` file and building the image. +### Code Server -```bash -make docker-compose.yml -make build -``` +The [code-server](https://github.com/cdr/code-server) container allows a user to +edit / debug their Drupal site from their browser. -At this point you could run it using `docker-compose`: +The code-server service can be disabled/enabled via the +`INCLUDE_CODE_SERVER_SERVICE` variable in your `.env` file. ```bash -docker-compose up -d +# Includes `code-server` as a service. +INCLUDE_CODE_SERVER_SERVICE=true ``` -To specify an image created outside of this repository, you can add the -following to `docker-compose.env.yml`: +* Run `make local`, `make up`, or `make demo` to build the containers and local file system(s). +* Then modify the `.env` file. +* Then `make pull` then `make up` to fetch the builds. +It will then report it created the **code-server** and recreated **traefik** and **drupal** containers. -```yaml -drupal: - image: YOUR_CUSTOM_IMAGE -``` +By default this will accessible at +[https://islandora.traefik.me:8443/](https://islandora.traefik.me:8443/). -## Services +If you do not have [secrets enabled](#secrets), you can login with the default +password: `password`. Otherwise you can find the password in the file +[./secrets/live/CODE_SERVER_PASSWORD](./secrets/live/CODE_SERVER_PASSWORD). -Islandora is composed of many different services, this project has split these -services up such that they have their own -[Container](https://www.docker.com/resources/what-container). +**N.B:** Do not expose this service on the internet without setting a strong +password via the `./secrets/live/CODE_SERVER_PASSWORD`, or better yet do not +expose it at all, and instead use port forward to access it if you have the +need. Exposing this service in an insecure way will allow root access to your +server to the public. -For in-depth documentation of the various `islandora` images see the -[isle-buildkit](https://github.com/Islandora-Devops/isle-buildkit) repository. +To enable xdebug for your request, you must also send an `XDEBUG_SESSION` cookie +with your request, this can be toggled on and off via a browser plugin such as +the following. -Other services will be documented below: +- [Chrome](https://chrome.google.com/webstore/detail/xdebug-helper/eadndfjplgieldjbigjakmdgkmoaaaoc?hl=en) +- [Firefox](https://addons.mozilla.org/en-GB/firefox/addon/xdebug-helper-for-firefox/) ### Watchtower @@ -415,19 +339,10 @@ INCLUDE_WATCHTOWER_SERVICE=true ### Traefik The [traefik](https://containo.us/traefik/) container acts as a reverse proxy, -and exposes some containers through port ``80``/``443``/``3306``. This allows access to the -following urls by default. - -- -- -- -- -- -- -- +and exposes some containers through port ``80``/``443``/``3306``. Since Drupal passes links to itself in the messages it passes to the microservices, -and occasionally other urls need to be resolved on containers that do not have +and occasionally other URLs need to be resolved on containers that do not have external access, we define aliases for most services on the internal network. Aliases like so are defined on most services to mimic their routing rules in @@ -462,6 +377,22 @@ It is not enabled by default. # Includes `etcd` as a service. INCLUDE_ETCD_SERVICE=false ``` +## Add Custom Makefile Commands +To add custom Makefile commands without adding upstream git conflict complexity, just create a new `custom.Makefile` and the Makefile will automatically include it. This can be a completely empty file that needs no header information. Just add a function in the following format. +```makefile +.PHONY: lowercasename +.SILENT: lowercasename +## This is the help description that comes up when using the 'make help` command. This needs to be placed with 2 # characters, after .PHONY & .SILENT but before the function call. And only take up a single line. +lowercasename: + echo "first line in command needs to be indented. There are exceptions to this, review functions in the Makefile for examples of these exceptions." +``` + +NOTE: A target you add in the custom.Makefile will not override an existing target with the same label in this repository's defautl Makefile. + +Running the new `custom.Makefile` commands are exactly the same as running any other Makefile command. Just run `make` and the function's name. +```bash +make lowercasename +``` ## Troubleshooting/Issues @@ -474,70 +405,7 @@ Post your questions here and subscribe for updates, meeting announcements, and t - [Islandora Group](https://groups.google.com/forum/?hl=en&fromgroups#!forum/islandora) - [Islandora Dev Group](https://groups.google.com/forum/?hl=en&fromgroups#!forum/islandora-dev) -## FAQ - -### Question: When doing an `drush config:import -y` I get one of the following errors - -**Error:** - -```html -Site UUID in source storage does not match the target Storage. -``` - -This occurs when your configuration has come from a site other than the one that -is currently installed. You can run the following command to override the site -`uuid` so that you can import your configuration: - -```bash -make set-site-uuid -``` - -**Error:** - -```html -Entities exist of type Shortcut link and Shortcut set -Default. These entities need to be deleted before importing -``` - -These are entities created by the `standard` installation profile, you can delete -them with the following command in the codebase folder: - -```bash -make delete-shortcut-entities -``` - -**Error:** - -```bash -Error: Call to a member function getConfigDependencyName() on null in ... Entity/EntityDisplayBase.php on line 325 #0 ... /codebase/web/core/lib/Drupal/Core/Config/Entity/ConfigEntityBase.php(318): Drupal\Core\Entity\EntityDisplayBase->calculateDependencies() -``` - -There is some bug in the dependencies of the various modules. Until those -dependencies issues are resolved just rerun the command until they go away. - -```bash -drush config:import -y -``` - -### Question: When doing an `drush sql:dump` I get one of the following errors - -**Error:** - -This arises from our use of [MariaDB] for the database in Docker, not matching -the same client on your host system. Which probably uses the [MySQL] clients -`mysqldump` executable. You can specify the following command using drush to get -around it: - -```bash -drush sql:dump --extra-dump="--column-statistics=0" > /tmp/dump.sql -``` - -Or you can use the provided method in the [Makefile](./Makefile) which should be -more portable. - -```bash -make database-dump DEST=/tmp/dump.sql -``` +For common errors, see `docs/troubleshooting.md`. ## Development diff --git a/build/demo-data/browse-collections.json b/build/demo-data/browse-collections.json new file mode 100644 index 000000000..f00a1dc11 --- /dev/null +++ b/build/demo-data/browse-collections.json @@ -0,0 +1,28 @@ +{ + "type": [ + { + "target_id": "page", + "target_type": "node_type" + } + ], + "status": [ + { + "value": true + } + ], + "title": [ + { + "value": "Browse Collections" + } + ], + "path": [ + { + "alias": "/browse-collections" + } + ], + "field_display_title": [ + { + "value": "Browse Collections" + } + ] +} diff --git a/build/demo-data/frontpage.html b/build/demo-data/frontpage.html new file mode 100644 index 000000000..a3507309f --- /dev/null +++ b/build/demo-data/frontpage.html @@ -0,0 +1,108 @@ +
+
+
+
+
+
+

+ HOPKINS DIGITAL LIBRARY +

+

+ Collections and scholarship from Johns Hopkins University +

+
+
+
+
+ +
+ +
+
+

Library Collections

+

+ Search digitized historic photographs, advertisements, texts, + and more from JHU's unique library collections +

+
+ +
+
+ +
+ +
+
+

JHU Scholarship

+

+ Find publications, theses, and dissertations by JHU faculty + and students +

+
+ +
+
+
+
+
+
+
+
+

+ The Hopkins Digital Library is the institutional repository of Johns + Hopkins University. It is a service of the + Sheridan Libraries that + uses digital technology to collect, preserve, and provide access to + unique library research collections as well as the intellectual and + creative outputs of Hopkins faculty, students, and academic staff. +

+

+ The digital materials are freely available to a worldwide audience + to advance discovery and innovation, and to support the research, + teaching, and public service missions of Johns Hopkins University +

+
+
+
+

Need more information?

+
+ +
+
+
+
+
+
+
\ No newline at end of file diff --git a/build/demo-data/homepage.json b/build/demo-data/homepage.json new file mode 100644 index 000000000..cee1f1fff --- /dev/null +++ b/build/demo-data/homepage.json @@ -0,0 +1,31 @@ +{ + "type": [ + { + "target_id": "home_page", + "target_type": "node_type" + } + ], + "status": [ + { + "value": true + } + ], + "title": [ + { + "value": "Islandora Sandbox" + } + ], + "body": [ + { + "value": "
\r\n\r\n
\r\n\r\n

Welcome to the Islandora Community! (Note that all communication in Islandora is subject to our code of conduct).

\r\n\r\n

This demo is loaded with sample content and configurations to demonstrate features of Islandora and provide a starting point for development.  Additional information is provided on the Islandora Website. 

\r\n\r\n

The Islandora Google groups mailing list is open to Islandora users who connect to share information. Join Islandora Slack to ask questions and learn more about community events where you can learn more about Islandora. Drupal also maintains a Slack community where you can ask more general questions about Drupal configuration. 

\r\n\r\n

We welcome you to visit and edit Islandora documentation or submit issues and code to Islandora via repositories in our Github organization.  

\r\n\r\n

Thanks, and happy building!

\r\n\r\n

login credentials

\r\n", + "format": "full_html", + "processed": "
\n\n
\n\n

Welcome to the Islandora Community! (Note that all communication in Islandora is subject to our code of conduct).

\n\n

This demo is loaded with sample content and configurations to demonstrate features of Islandora and provide a starting point for development.  Additional information is provided on the Islandora Website. 

\n\n

The Islandora Google groups mailing list is open to Islandora users who connect to share information. Join Islandora Slack to ask questions and learn more about community events where you can learn more about Islandora. Drupal also maintains a Slack community where you can ask more general questions about Drupal configuration. 

\n\n

We welcome you to visit and edit Islandora documentation or submit issues and code to Islandora via repositories in our Github organization.  

\n\n

Thanks, and happy building!

\n\n

login credentials

\n", + "summary": "" + } + ], + "field_display_title": [ + { + "value": "Islandora Sandbox" + } + ] +} diff --git a/build/demo-data/jhu_homepage.json b/build/demo-data/jhu_homepage.json new file mode 100644 index 000000000..a7a090cee --- /dev/null +++ b/build/demo-data/jhu_homepage.json @@ -0,0 +1,31 @@ +{ + "type": [ + { + "target_id": "home_page", + "target_type": "node_type" + } + ], + "status": [ + { + "value": true + } + ], + "title": [ + { + "value": "HOPKINS DIGITAL LIBRARY" + } + ], + "body": [ + { + "value": "

HOPKINS DIGITAL LIBRARY

Collections and scholarship from Johns Hopkins University

Library Collections

Search digitized historic photographs, advertisements, texts,and more from JHU's unique library collections

JHU Scholarship

Find publications, theses, and dissertations by JHU facultyand students

The Hopkins Digital Library is the institutional repository of JohnsHopkins University. It is a service of theSheridan Libraries thatuses digital technology to collect, preserve, and provide access tounique library research collections as well as the intellectual andcreative outputs of Hopkins faculty, students, and academic staff.

The digital materials are freely available to a worldwide audienceto advance discovery and innovation, and to support the research,teaching, and public service missions of Johns Hopkins University

Need more information?

Contact Us
", + "format": "full_html", + "processed": "

HOPKINS DIGITAL LIBRARY

Collections and scholarship from Johns Hopkins University

Library Collections

Search digitized historic photographs, advertisements, texts,and more from JHU's unique library collections

JHU Scholarship

Find publications, theses, and dissertations by JHU facultyand students

The Hopkins Digital Library is the institutional repository of JohnsHopkins University. It is a service of theSheridan Libraries thatuses digital technology to collect, preserve, and provide access tounique library research collections as well as the intellectual andcreative outputs of Hopkins faculty, students, and academic staff.

The digital materials are freely available to a worldwide audienceto advance discovery and innovation, and to support the research,teaching, and public service missions of Johns Hopkins University

Need more information?

Contact Us
", + "summary": "" + } + ], + "field_display_title": [ + { + "value": "HOPKINS DIGITAL LIBRARY" + } + ] +} diff --git a/build/docker-compose/docker-compose.acme.yml b/build/docker-compose/docker-compose.acme.yml new file mode 100644 index 000000000..8fb4c5196 --- /dev/null +++ b/build/docker-compose/docker-compose.acme.yml @@ -0,0 +1,45 @@ +version: "3.7" +services: + traefik: + # Do not set `api.insecure`, `api.dashboard`, `api.debug` to `true` in production. + # Also do not expose database 3306/5432, as an entry point. + # If the commands below are changed, please copy the changes to `docker-compose.acme.yml`. + command: + - --api.insecure=${EXPOSE_TRAEFIK_DASHBOARD:-false} + - --api.dashboard=${EXPOSE_TRAEFIK_DASHBOARD:-false} + - --api.debug=${EXPOSE_TRAEFIK_DASHBOARD:-false} + - --entryPoints.http.address=:80 + - --entryPoints.https.address=:443 + - --entryPoints.mysql.address=:3306 + - --entryPoints.postgresql.address=:5432 + - --entryPoints.fcrepo.address=:8081 + - --entryPoints.blazegraph.address=:8082 + - --entryPoints.activemq.address=:8161 + - --entryPoints.solr.address=:8983 + - --entryPoints.code-server.address=:8443 + - --log.level=${TRAEFIK_LOG_LEVEL:-ERROR} + - --providers.docker + - --providers.docker.network=gateway + - --providers.docker.exposedByDefault=false + - --providers.file.filename=/etc/traefik/tls.yml + - '--providers.docker.defaultRule=Host(`${DOMAIN}`)' + - --certificatesresolvers.myresolver.acme.httpchallenge=true + - --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=http + - --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL-your-email@example.com} + - --certificatesresolvers.myresolver.acme.keyType=${ACME_KEY_TYPE-RSA4096} + - --certificatesresolvers.myresolver.acme.storage=/acme/acme.json + - --certificatesResolvers.myresolver.acme.caServer=${ACME_SERVER-https://acme-v02.api.letsencrypt.org/directory} + - --certificatesresolvers.myresolver.acme.certificatesduration=${ACME_CERT_DURATION-2160} + - --certificatesresolvers.myresolver.acme.eab.kid=${ACME_EAB_KID-} + - --certificatesresolvers.myresolver.acme.eab.hmacencoded=${ACME_EAB_HMAC-} + volumes: + - ../../acme:/acme:rw + cantaloupe: + labels: + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_https.tls.certresolver=myresolver + drupal: + labels: + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.tls.certresolver=myresolver + matomo: + labels: + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_https.tls.certresolver=myresolver diff --git a/docker-compose.activemq.yml b/build/docker-compose/docker-compose.activemq.yml similarity index 67% rename from docker-compose.activemq.yml rename to build/docker-compose/docker-compose.activemq.yml index 74a4281e6..35f7337fa 100644 --- a/docker-compose.activemq.yml +++ b/build/docker-compose/docker-compose.activemq.yml @@ -3,21 +3,26 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: activemq-data: services: activemq: image: ${REPOSITORY:-islandora}/activemq:${TAG:-latest} + restart: ${RESTART_POLICY:-unless-stopped} volumes: - activemq-data:/opt/activemq/data labels: # Do not expose in production. - - traefik.enable=true + - traefik.enable=${EXPOSE_ACTIVEMQ:-false} - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-activemq.loadbalancer.server.port=8161 - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-activemq_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-activemq - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-activemq_http.entrypoints=http + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-activemq_http.entrypoints=activemq networks: default: - gateway: # Do not expose in production. + deploy: + resources: + limits: + memory: ${ACTIVEMQ_MEMORY_LIMIT:-2G} + reservations: + memory: 1G diff --git a/build/docker-compose/docker-compose.alpaca.yml b/build/docker-compose/docker-compose.alpaca.yml new file mode 100644 index 000000000..0d75afab0 --- /dev/null +++ b/build/docker-compose/docker-compose.alpaca.yml @@ -0,0 +1,19 @@ +version: "3.7" +networks: + default: + internal: true +services: + alpaca: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/alpaca:${TAG:-latest} + environment: + ALPACA_OCR_TIMEOUT: ${ALPACA_OCR_TIMEOUT:-10000} + ALPACA_HOUDINI_TIMEOUT: ${ALPACA_HOUDINI_TIMEOUT:-10000} + ALPACA_HOMARUS_TIMEOUT: ${ALPACA_HOMARUS_TIMEOUT:-10000} + ALPACA_FITS_TIMEOUT: ${ALPACA_HOMARUS_TIMEOUT:-10000} + deploy: + resources: + limits: + memory: ${ALPACA_MEMORY_LIMIT:-2G} + reservations: + memory: 1G \ No newline at end of file diff --git a/docker-compose.blazegraph.yml b/build/docker-compose/docker-compose.blazegraph.yml similarity index 54% rename from docker-compose.blazegraph.yml rename to build/docker-compose/docker-compose.blazegraph.yml index a92cf9fa3..05f733376 100644 --- a/docker-compose.blazegraph.yml +++ b/build/docker-compose/docker-compose.blazegraph.yml @@ -3,24 +3,25 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: blazegraph-data: services: blazegraph: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/blazegraph:${TAG:-latest} volumes: - blazegraph-data:/data networks: default: - # Allow services to use the edge name to reference this service. - aliases: - - blazegraph-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - - blazegraph-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} - gateway: # Do not expose in production. labels: - - traefik.enable=true - - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-blazegraph.loadbalancer.server.port=80 + - traefik.enable=${EXPOSE_BLAZEGRAPH:-false} + - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-blazegraph.loadbalancer.server.port=8080 - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-blazegraph_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-blazegraph - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-blazegraph_http.entrypoints=http + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-blazegraph_http.entrypoints=blazegraph + deploy: + resources: + limits: + memory: ${BLAZEGRAPH_MEMORY_LIMIT:-5G} + reservations: + memory: 2G \ No newline at end of file diff --git a/docker-compose.cantaloupe.yml b/build/docker-compose/docker-compose.cantaloupe.yml similarity index 60% rename from docker-compose.cantaloupe.yml rename to build/docker-compose/docker-compose.cantaloupe.yml index 9502bdec2..6c09304fd 100644 --- a/docker-compose.cantaloupe.yml +++ b/build/docker-compose/docker-compose.cantaloupe.yml @@ -3,27 +3,36 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: cantaloupe-data: services: cantaloupe: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/cantaloupe:${TAG:-latest} + environment: + CANTALOUPE_DELEGATE_SCRIPT_ENABLED: ${CANTALOUPE_DELEGATE_SCRIPT_ENABLED} # "true" + CANTALOUPE_DELEGATE_SCRIPT_PATHNAME: ${CANTALOUPE_DELEGATE_SCRIPT_PATHNAME} #/opt/tomcat/bin/delegates.rb + CANTALOUPE_HTTPSOURCE_LOOKUP_STRATEGY: ${CANTALOUPE_HTTPSOURCE_LOOKUP_STRATEGY} # ScriptLookupStrategy volumes: - cantaloupe-data:/data labels: - - traefik.enable=true - - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe.loadbalancer.server.port=80 + - traefik.enable=${EXPOSE_CANTALOUPE:-true} + - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe.loadbalancer.server.port=8080 - traefik.http.middlewares.cantaloupe-redirectscheme.redirectscheme.scheme=https - traefik.http.middlewares.cantaloupe-redirectscheme.redirectscheme.permanent=true - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_http.entrypoints=http - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_http.rule=Host(`islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}`,`islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}`) && PathPrefix(`/cantaloupe`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_http.rule=Host(`${DOMAIN}`) && PathPrefix(`/cantaloupe`) - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_http.middlewares=cantaloupe-redirectscheme - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_https.entrypoints=https - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_https.rule=Host(`islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}`,`islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}`) && PathPrefix(`/cantaloupe`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_https.rule=Host(`${DOMAIN}`) && PathPrefix(`/cantaloupe`) - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-cantaloupe_https.tls=true networks: default: - gateway: + deploy: + resources: + limits: + memory: ${CANTALOUPE_MEMORY_LIMIT:-5G} + reservations: + memory: 1G diff --git a/build/docker-compose/docker-compose.code-server.yml b/build/docker-compose/docker-compose.code-server.yml new file mode 100644 index 000000000..28528900c --- /dev/null +++ b/build/docker-compose/docker-compose.code-server.yml @@ -0,0 +1,92 @@ +# Creates the drupal container. +# +# References: +# - https://www.drupal.org/docs +version: '3.7' +networks: + default: + internal: true + gateway: + external: true +secrets: + CODE_SERVER_PASSWORD: + file: "../../secrets/live/CODE_SERVER_PASSWORD" +services: + code-server: + environment: + PHP_MEMORY_LIMIT: ${PHP_MEMORY_LIMIT} + PHP_POST_MAX_SIZE: ${PHP_POST_MAX_SIZE} + PHP_UPLOAD_MAX_FILESIZE: ${PHP_UPLOAD_MAX_FILESIZE} + PHP_MAX_EXECUTION_TIME: ${PHP_MAX_EXECUTION_TIME} + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/code-server:${TAG:-latest} + labels: + - traefik.enable=${EXPOSE_CODE_SERVER:-false} + # code-server + - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-code-server.loadbalancer.server.port=8443 + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-code-server_https.service=${COMPOSE_PROJECT_NAME-isle-dc}-code-server + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-code-server_https.entrypoints=code-server + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-code-server_https.rule=Host(`${DOMAIN}`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-code-server_https.tls=true + # drupal + - traefik.http.middlewares.drupal-redirectscheme.redirectscheme.scheme=https + - traefik.http.middlewares.drupal-redirectscheme.redirectscheme.permanent=true + - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-drupal.loadbalancer.server.port=80 + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-drupal + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.entrypoints=http + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.rule=Host(`${DOMAIN}`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.middlewares=drupal-redirectscheme + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.service=${COMPOSE_PROJECT_NAME-isle-dc}-drupal + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.entrypoints=https + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.rule=Host(`${DOMAIN}`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.tls=true + secrets: + - CODE_SERVER_PASSWORD + volumes: + # Mount and serve contents of Drupal site. + - type: volume + source: drupal-root + target: /var/www/drupal + volume: + nocopy: true + # Mount and serve Drupal files. + - type: volume + source: drupal-sites-data + target: /var/www/drupal/web/sites/default/files + volume: + nocopy: true + # Volumes for code-server cache. + - code-server-data:/opt/code-server/data + networks: + default: + gateway: + depends_on: + - drupal + deploy: + resources: + limits: + memory: ${IDE_MEMORY_LIMIT:-5G} + reservations: + memory: 2G + drupal: + environment: + PHP_MEMORY_LIMIT: ${PHP_MEMORY_LIMIT} + PHP_POST_MAX_SIZE: ${PHP_POST_MAX_SIZE} + PHP_UPLOAD_MAX_FILESIZE: ${PHP_UPLOAD_MAX_FILESIZE} + PHP_MAX_EXECUTION_TIME: ${PHP_MAX_EXECUTION_TIME} + # Disable traefik for Drupal as code-server will respond to all requests. + labels: + - traefik.enable=false + volumes: + # Allow code-server to serve Drupal / override it. + - drupal-root:/var/www/drupal + deploy: + resources: + limits: + memory: ${DRUPAL_MEMORY_LIMIT:-5G} + reservations: + memory: 2G +volumes: + drupal-root: {} + drupal-sites-data: {} + code-server-data: {} diff --git a/build/docker-compose/docker-compose.crayfish.yml b/build/docker-compose/docker-compose.crayfish.yml new file mode 100644 index 000000000..312466ab6 --- /dev/null +++ b/build/docker-compose/docker-compose.crayfish.yml @@ -0,0 +1,57 @@ +version: "3.7" +networks: + default: + internal: true + gateway: + external: true +services: + homarus: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/homarus:${TAG:-latest} + deploy: + resources: + limits: + memory: ${HOMARUS_MEMORY_LIMIT:-1G} + reservations: + memory: 512M + houdini: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/houdini:${TAG:-latest} + deploy: + resources: + limits: + memory: ${HOUDINI_MEMORY_LIMIT:-1G} + reservations: + memory: 512M + hypercube: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/hypercube:${TAG:-latest} + deploy: + resources: + limits: + memory: ${HYPERCUBE_MEMORY_LIMIT:-512M} + reservations: + memory: 256M + milliner: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/milliner:${TAG:-latest} + environment: + MILLINER_FEDORA6: ${FEDORA_6} + networks: + default: + gateway: + deploy: + resources: + limits: + memory: ${MILLINER_MEMORY_LIMIT:-1G} + reservations: + memory: 512M + recast: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/recast:${TAG:-latest} + deploy: + resources: + limits: + memory: ${RECAST_MEMORY_LIMIT:-1G} + reservations: + memory: 512M diff --git a/build/docker-compose/docker-compose.crayfits.yml b/build/docker-compose/docker-compose.crayfits.yml new file mode 100644 index 000000000..2f31c657b --- /dev/null +++ b/build/docker-compose/docker-compose.crayfits.yml @@ -0,0 +1,25 @@ +version: "3.7" +networks: + default: + internal: true +services: + crayfits: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/crayfits:${TAG:-latest} + depends_on: + - fits + deploy: + resources: + limits: + memory: ${CRAYFITS_MEMORY_LIMIT:-1G} + reservations: + memory: 512M + fits: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/fits:${TAG:-latest} + deploy: + resources: + limits: + memory: ${FITS_MEMORY_LIMIT:-2G} + reservations: + memory: 1000M \ No newline at end of file diff --git a/docker-compose.custom.yml b/build/docker-compose/docker-compose.custom.yml similarity index 93% rename from docker-compose.custom.yml rename to build/docker-compose/docker-compose.custom.yml index 0c9b090de..83dcdccf0 100644 --- a/docker-compose.custom.yml +++ b/build/docker-compose/docker-compose.custom.yml @@ -4,8 +4,7 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: drupal-sites-data: solr-data: @@ -13,7 +12,7 @@ services: # The service name is drupal that is the default host name used by micro-services etc. drupal: build: - context: ./ + context: ../../ dockerfile: ${PROJECT_DRUPAL_DOCKERFILE:-./Dockerfile} environment: # @@ -33,6 +32,6 @@ services: solr: volumes: # On a production site you may not want to take this approach but instead refer to each of the cores - # data directories specifically and maintain the configuration as part of a customized image, where + # data directories specifically and maintain the configuration as part of a customized image, where # in your configuration is Solr managed under source control somewhere. - solr-data:/opt/solr/server/solr diff --git a/docker-compose.demo.yml b/build/docker-compose/docker-compose.demo.yml similarity index 90% rename from docker-compose.demo.yml rename to build/docker-compose/docker-compose.demo.yml index cd7e3f5f1..9bd0074fc 100644 --- a/docker-compose.demo.yml +++ b/build/docker-compose/docker-compose.demo.yml @@ -10,14 +10,14 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: drupal-sites-data: solr-data: services: # The service name is drupal that is the default host name used by micro-services etc. drupal: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/demo:${TAG:-latest} volumes: - drupal-sites-data:/var/www/drupal/web/sites/default/files @@ -32,8 +32,9 @@ services: - blazegraph # Extends docker-compose.solr.yml so that they share a volume. solr: + restart: ${RESTART_POLICY:-unless-stopped} volumes: # On a production site you may not want to take this approach but instead refer to each of the cores - # data directories specifically and maintain the configuration as part of a customized image, where + # data directories specifically and maintain the configuration as part of a customized image, where # in your configuration is Solr managed under source control somewhere. - solr-data:/opt/solr/server/solr diff --git a/docker-compose.crayfish.mariadb.yml b/build/docker-compose/docker-compose.drupal.mariadb.yml similarity index 53% rename from docker-compose.crayfish.mariadb.yml rename to build/docker-compose/docker-compose.drupal.mariadb.yml index 6752132fa..30703bcd8 100644 --- a/docker-compose.crayfish.mariadb.yml +++ b/build/docker-compose/docker-compose.drupal.mariadb.yml @@ -1,8 +1,6 @@ version: "3.7" services: # Override defaults so Mariadb is used as the database for this service. - gemini: + drupal: environment: - GEMINI_DB_DRIVER: pdo_mysql - GEMINI_DB_HOST: mariadb - GEMINI_DB_PORT: 3306 + DRUPAL_DEFAULT_DB_DRIVER: mysql \ No newline at end of file diff --git a/docker-compose.crayfish.postgresql.yml b/build/docker-compose/docker-compose.drupal.postgresql.yml similarity index 53% rename from docker-compose.crayfish.postgresql.yml rename to build/docker-compose/docker-compose.drupal.postgresql.yml index ac2d90757..40d16db6b 100644 --- a/docker-compose.crayfish.postgresql.yml +++ b/build/docker-compose/docker-compose.drupal.postgresql.yml @@ -1,8 +1,6 @@ version: "3.7" services: # Override defaults so PostgreSQL is used as the database for this service. - gemini: + drupal: environment: - GEMINI_DB_DRIVER: pdo_pgsql - GEMINI_DB_HOST: postgresql - GEMINI_DB_PORT: 5432 + DRUPAL_DEFAULT_DB_DRIVER: postgresql diff --git a/docker-compose.drupal.yml b/build/docker-compose/docker-compose.drupal.yml similarity index 58% rename from docker-compose.drupal.yml rename to build/docker-compose/docker-compose.drupal.yml index 60adb9607..52b3f0bea 100644 --- a/docker-compose.drupal.yml +++ b/build/docker-compose/docker-compose.drupal.yml @@ -4,28 +4,41 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true services: # The service name is drupal that is the default host name used by micro-services etc. # Needs to match against demo, custom, and local. drupal: + restart: ${RESTART_POLICY:-unless-stopped} + environment: + DRUPAL_DEFAULT_CANTALOUPE_URL: https://${DOMAIN}/cantaloupe/iiif/2 + DRUPAL_DEFAULT_FCREPO_HOST: ${DOMAIN} + DRUPAL_DEFAULT_FCREPO_PORT: 8081 + DRUPAL_DEFAULT_MATOMO_URL: https://${DOMAIN}/matomo/ + DRUPAL_DEFAULT_SITE_URL: http://${DOMAIN} # Make sure this is just http and not https! + DRUPAL_DEFAULT_PROFILE: ${DRUPAL_INSTALL_PROFILE} + PHP_MEMORY_LIMIT: ${PHP_MEMORY_LIMIT} + PHP_POST_MAX_SIZE: ${PHP_POST_MAX_SIZE} + PHP_UPLOAD_MAX_FILESIZE: ${PHP_UPLOAD_MAX_FILESIZE} + PHP_MAX_EXECUTION_TIME: ${PHP_MAX_EXECUTION_TIME} labels: - - traefik.enable=true + - traefik.enable=${EXPOSE_DRUPAL:-true} - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-drupal.loadbalancer.server.port=80 - traefik.http.middlewares.drupal-redirectscheme.redirectscheme.scheme=https - traefik.http.middlewares.drupal-redirectscheme.redirectscheme.permanent=true - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-drupal - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.entrypoints=http - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.rule=Host(`islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}`,`islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.rule=Host(`${DOMAIN}`) - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_http.middlewares=drupal-redirectscheme - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.entrypoints=https - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.rule=Host(`islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}`,`islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.rule=Host(`${DOMAIN}`) - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-drupal_https.tls=true networks: default: - # Allow services (like Matomo) to use the edge name to reference this service in addition to `drupal`. - aliases: - - islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - - islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} gateway: + deploy: + resources: + limits: + memory: ${DRUPAL_MEMORY_LIMIT:-512M} + reservations: + memory: 256M \ No newline at end of file diff --git a/docker-compose.etcd.yml b/build/docker-compose/docker-compose.etcd.yml similarity index 94% rename from docker-compose.etcd.yml rename to build/docker-compose/docker-compose.etcd.yml index fed7d3de3..25f123de7 100644 --- a/docker-compose.etcd.yml +++ b/build/docker-compose/docker-compose.etcd.yml @@ -14,6 +14,7 @@ services: # # @todo copy commands folder over to isle-dc? etcd: + restart: ${RESTART_POLICY:-unless-stopped} image: gcr.io/etcd-development/etcd:v3.4.7 environment: ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379" diff --git a/docker-compose.fcrepo.mariadb.yml b/build/docker-compose/docker-compose.fcrepo.mariadb.yml similarity index 57% rename from docker-compose.fcrepo.mariadb.yml rename to build/docker-compose/docker-compose.fcrepo.mariadb.yml index 24df7094e..14cd6f83b 100644 --- a/docker-compose.fcrepo.mariadb.yml +++ b/build/docker-compose/docker-compose.fcrepo.mariadb.yml @@ -3,6 +3,4 @@ services: # Override defaults so Mariadb is used as the database for this service. fcrepo: environment: - FCREPO_PERSISTENCE_TYPE: mysql - FCREPO_DB_HOST: mariadb - FCREPO_DB_PORT: 3306 + FCREPO_PERSISTENCE_TYPE: mysql \ No newline at end of file diff --git a/docker-compose.fcrepo.postgresql.yml b/build/docker-compose/docker-compose.fcrepo.postgresql.yml similarity index 74% rename from docker-compose.fcrepo.postgresql.yml rename to build/docker-compose/docker-compose.fcrepo.postgresql.yml index afcb30fac..b4c272363 100644 --- a/docker-compose.fcrepo.postgresql.yml +++ b/build/docker-compose/docker-compose.fcrepo.postgresql.yml @@ -4,5 +4,3 @@ services: fcrepo: environment: FCREPO_PERSISTENCE_TYPE: postgresql - FCREPO_DB_HOST: postgresql - FCREPO_DB_PORT: 5432 diff --git a/build/docker-compose/docker-compose.fcrepo.yml b/build/docker-compose/docker-compose.fcrepo.yml new file mode 100644 index 000000000..74a2ab407 --- /dev/null +++ b/build/docker-compose/docker-compose.fcrepo.yml @@ -0,0 +1,30 @@ +version: "3.7" +networks: + default: + internal: true + gateway: + external: true +volumes: + fcrepo-data: +services: + fcrepo: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/fcrepo:${TAG:-latest} + environment: + FCREPO_ALLOW_EXTERNAL_DRUPALHTTP: http://${DOMAIN}/ + FCREPO_ALLOW_EXTERNAL_DRUPALHTTPS: https://${DOMAIN}/ + FCREPO_TOMCAT_ADMIN_ROLES: manager-gui,fedoraAdmin + FCREPO_TOMCAT_ADMIN_USER: admin + FCREPO_DISABLE_SYN: ${DISABLE_SYN} + volumes: + - fcrepo-data:/data + depends_on: + - activemq + networks: + default: + labels: + # Do not expose in production. + - traefik.enable=${EXPOSE_FEDORA:-false} + - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo.loadbalancer.server.port=8080 + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo_http.entrypoints=fcrepo diff --git a/build/docker-compose/docker-compose.fcrepo6.mariadb.yml b/build/docker-compose/docker-compose.fcrepo6.mariadb.yml new file mode 100644 index 000000000..14cd6f83b --- /dev/null +++ b/build/docker-compose/docker-compose.fcrepo6.mariadb.yml @@ -0,0 +1,6 @@ +version: "3.7" +services: + # Override defaults so Mariadb is used as the database for this service. + fcrepo: + environment: + FCREPO_PERSISTENCE_TYPE: mysql \ No newline at end of file diff --git a/build/docker-compose/docker-compose.fcrepo6.postgresql.yml b/build/docker-compose/docker-compose.fcrepo6.postgresql.yml new file mode 100644 index 000000000..b4c272363 --- /dev/null +++ b/build/docker-compose/docker-compose.fcrepo6.postgresql.yml @@ -0,0 +1,6 @@ +version: "3.7" +services: + # Override defaults so PostgreSQL is used as the database for this service. + fcrepo: + environment: + FCREPO_PERSISTENCE_TYPE: postgresql diff --git a/build/docker-compose/docker-compose.fcrepo6.yml b/build/docker-compose/docker-compose.fcrepo6.yml new file mode 100644 index 000000000..13d5c820c --- /dev/null +++ b/build/docker-compose/docker-compose.fcrepo6.yml @@ -0,0 +1,36 @@ +version: "3.7" +networks: + default: + internal: true + gateway: + external: true +volumes: + fcrepo-data: +services: + fcrepo: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/fcrepo6:${TAG:-latest} + environment: + FCREPO_ALLOW_EXTERNAL_DRUPALHTTP: http://${DOMAIN}/ + FCREPO_ALLOW_EXTERNAL_DRUPALHTTPS: https://${DOMAIN}/ + FCREPO_TOMCAT_ADMIN_ROLES: manager-gui,fedoraAdmin + FCREPO_TOMCAT_ADMIN_USER: admin + FCREPO_DISABLE_SYN: ${DISABLE_SYN} + volumes: + - fcrepo-data:/data + depends_on: + - activemq + networks: + default: + labels: + # Do not expose in production. + - traefik.enable=${EXPOSE_FEDORA:-false} + - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo.loadbalancer.server.port=8080 + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo_http.entrypoints=fcrepo + deploy: + resources: + limits: + memory: ${FCREPO_MEMORY_LIMIT:-2G} + reservations: + memory: 1G diff --git a/docker-compose.local.yml b/build/docker-compose/docker-compose.local.yml similarity index 77% rename from docker-compose.local.yml rename to build/docker-compose/docker-compose.local.yml index d8673b562..f4f44103d 100644 --- a/docker-compose.local.yml +++ b/build/docker-compose/docker-compose.local.yml @@ -9,18 +9,21 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: drupal-sites-data: solr-data: services: drupal: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/drupal:${TAG:-latest} volumes: - - ./codebase:/var/www/drupal:delegated + - ../../codebase:/var/www/drupal:delegated - drupal-sites-data:/var/www/drupal/web/sites/default/files - solr-data:/opt/solr/server/solr + environment: + DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG: ${INSTALL_EXISTING_CONFIG} + DRUPAL_DEFAULT_PROFILE: ${DRUPAL_INSTALL_PROFILE} depends_on: # Requires a the very minimum a database. - ${DRUPAL_DATABASE_SERVICE} @@ -28,6 +31,6 @@ services: solr: volumes: # On a production site you may not want to take this approach but instead refer to each of the cores - # data directories specifically and maintain the configuration as part of a customized image, where + # data directories specifically and maintain the configuration as part of a customized image, where # in your configuration is Solr managed under source control somewhere. - solr-data:/opt/solr/server/solr diff --git a/docker-compose.mariadb.yml b/build/docker-compose/docker-compose.mariadb.yml similarity index 70% rename from docker-compose.mariadb.yml rename to build/docker-compose/docker-compose.mariadb.yml index d3cd7677d..ee6ca9c14 100644 --- a/docker-compose.mariadb.yml +++ b/build/docker-compose/docker-compose.mariadb.yml @@ -9,6 +9,7 @@ services: # Always included even if PostgreSQL is included as Matomo only works with MySQL. # https://github.com/matomo-org/matomo/issues/500 mariadb: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/mariadb:${TAG:-latest} volumes: - mariadb-data:/var/lib/mysql @@ -19,18 +20,16 @@ services: # Since this is not http, but tcp traffic it does does not understand the concept of a "host". # so we must dedicate a port to it in traefik, and direct all traffic to this router: HostSNI(`*`). labels: - - traefik.enable=true + - traefik.enable=${EXPOSE_MYSQL:-false} - traefik.tcp.services.${COMPOSE_PROJECT_NAME-isle-dc}-mysql.loadbalancer.server.port=3306 - traefik.tcp.routers.${COMPOSE_PROJECT_NAME-isle-dc}-mysql_tcp.service=${COMPOSE_PROJECT_NAME-isle-dc}-mysql - traefik.tcp.routers.${COMPOSE_PROJECT_NAME-isle-dc}-mysql_tcp.entrypoints=mysql - traefik.tcp.routers.${COMPOSE_PROJECT_NAME-isle-dc}-mysql_tcp.rule=HostSNI(`*`) networks: default: - # Allow drupal to access the database with it's edge name to reference - # this service in addition to `database`. This allows us to use the edge - # name in settings.php so Drush on the host machine can be used in the - # codebase folder. - aliases: - - mariadb-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - - mariadb-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} - gateway: + deploy: + resources: + limits: + memory: ${MARIADB_MEMORY_LIMIT:-1G} + reservations: + memory: 512M \ No newline at end of file diff --git a/docker-compose.matomo.yml b/build/docker-compose/docker-compose.matomo.yml similarity index 81% rename from docker-compose.matomo.yml rename to build/docker-compose/docker-compose.matomo.yml index da625f9bd..be12f2ddd 100644 --- a/docker-compose.matomo.yml +++ b/build/docker-compose/docker-compose.matomo.yml @@ -3,34 +3,41 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true volumes: matomo-config-data: services: matomo: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/matomo:${TAG:-latest} + environment: + MATOMO_DEFAULT_HOST: ${DOMAIN} volumes: - matomo-config-data:/var/www/matomo depends_on: - mariadb networks: default: - gateway: # Needs external access to request plugins, etc. labels: # Do not expose in production over http, setup https. - - traefik.enable=true + - traefik.enable=${EXPOSE_MATOMO:-true} - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-matomo.loadbalancer.server.port=80 - traefik.http.middlewares.${COMPOSE_PROJECT_NAME-isle-dc}-matomo-redirectscheme.redirectscheme.scheme=https - traefik.http.middlewares.${COMPOSE_PROJECT_NAME-isle-dc}-matomo-redirectscheme.redirectscheme.permanent=true - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-matomo - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_http.entrypoints=http - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_http.rule=Host(`islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}`,`islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}`) && PathPrefix(`/matomo`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_http.rule=Host(`${DOMAIN}`) && PathPrefix(`/matomo`) - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_http.middlewares=${COMPOSE_PROJECT_NAME-isle-dc}-matomo-redirectscheme - traefik.http.middlewares.${COMPOSE_PROJECT_NAME-isle-dc}-matomo-stripprefix.stripprefix.prefixes=/matomo - traefik.http.middlewares.${COMPOSE_PROJECT_NAME-isle-dc}-matomo-customrequestheaders.headers.customrequestheaders.X-Forwarded-Uri=/matomo - traefik.http.middlewares.${COMPOSE_PROJECT_NAME-isle-dc}-matomo.chain.middlewares=${COMPOSE_PROJECT_NAME-isle-dc}-matomo-stripprefix,${COMPOSE_PROJECT_NAME-isle-dc}-matomo-customrequestheaders - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_https.entrypoints=https - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_https.rule=Host(`islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}`,`islandora-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me}`) && PathPrefix(`/matomo`) + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_https.rule=Host(`${DOMAIN}`) && PathPrefix(`/matomo`) - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_https.tls=true - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-matomo_https.middlewares=${COMPOSE_PROJECT_NAME-isle-dc}-matomo + deploy: + resources: + limits: + memory: ${MATOMO_MEMORY_LIMIT:-1G} + reservations: + memory: 512M diff --git a/docker-compose.postgresql.yml b/build/docker-compose/docker-compose.postgresql.yml similarity index 95% rename from docker-compose.postgresql.yml rename to build/docker-compose/docker-compose.postgresql.yml index bfc51ec60..ee95d2102 100644 --- a/docker-compose.postgresql.yml +++ b/build/docker-compose/docker-compose.postgresql.yml @@ -14,6 +14,7 @@ services: # are included when the respective 'SERVICENAME_DATABASE_SERVICE' variable is set # to 'postgresql'. postgresql: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/postgresql:${TAG:-latest} volumes: - postgresql-data:/var/lib/postgresql/data @@ -23,7 +24,7 @@ services: # Since this is not http, but tcp traffic it does does not understand the concept of a "host". # so we must dedicate a port to it in traefik, and direct all traffic to this router: HostSNI(`*`). labels: - - traefik.enable=true + - traefik.enable=${EXPOSE_POSTGRES:-false} - traefik.tcp.services.${COMPOSE_PROJECT_NAME-isle-dc}-postgresql.loadbalancer.server.port=5432 - traefik.tcp.routers.${COMPOSE_PROJECT_NAME-isle-dc}-postgresql_tcp.service=${COMPOSE_PROJECT_NAME-isle-dc}-postgresql - traefik.tcp.routers.${COMPOSE_PROJECT_NAME-isle-dc}-postgresql_tcp.entrypoints=postgresql @@ -38,4 +39,3 @@ services: - postgresql-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - postgresql-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} gateway: - diff --git a/build/docker-compose/docker-compose.secrets.yml b/build/docker-compose/docker-compose.secrets.yml new file mode 100644 index 000000000..142b56658 --- /dev/null +++ b/build/docker-compose/docker-compose.secrets.yml @@ -0,0 +1,89 @@ +version: "3.7" +secrets: + ACTIVEMQ_PASSWORD: + file: "../../secrets/live/ACTIVEMQ_PASSWORD" + ACTIVEMQ_WEB_ADMIN_PASSWORD: + file: "../../secrets/live/ACTIVEMQ_WEB_ADMIN_PASSWORD" + ALPACA_ACTIVEMQ_PASSWORD: + file: "../../secrets/live/ALPACA_ACTIVEMQ_PASSWORD" + ALPACA_KARAF_ADMIN_PASSWORD: + file: "../../secrets/live/ALPACA_KARAF_ADMIN_PASSWORD" + DB_ROOT_PASSWORD: + file: "../../secrets/live/DB_ROOT_PASSWORD" + DRUPAL_DEFAULT_ACCOUNT_PASSWORD: + file: "../../secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD" + DRUPAL_DEFAULT_CONFIGDIR: + file: "../../secrets/live/DRUPAL_DEFAULT_CONFIGDIR" + DRUPAL_DEFAULT_DB_PASSWORD: + file: "../../secrets/live/DRUPAL_DEFAULT_DB_PASSWORD" + DRUPAL_DEFAULT_SALT: + file: "../../secrets/live/DRUPAL_DEFAULT_SALT" + FCREPO_DB_PASSWORD: + file: "../../secrets/live/FCREPO_DB_PASSWORD" + JWT_ADMIN_TOKEN: + file: "../../secrets/live/JWT_ADMIN_TOKEN" + JWT_PUBLIC_KEY: + file: "../../secrets/live/JWT_PUBLIC_KEY" + JWT_PRIVATE_KEY: + file: "../../secrets/live/JWT_PRIVATE_KEY" + MATOMO_DB_PASSWORD: + file: "../../secrets/live/MATOMO_DB_PASSWORD" + MATOMO_USER_PASS: + file: "../../secrets/live/MATOMO_USER_PASS" + TOMCAT_ADMIN_PASSWORD: + file: "../../secrets/live/TOMCAT_ADMIN_PASSWORD" +services: + activemq: + secrets: + - ACTIVEMQ_PASSWORD + - ACTIVEMQ_WEB_ADMIN_PASSWORD + alpaca: + secrets: + - ALPACA_ACTIVEMQ_PASSWORD + - ALPACA_KARAF_ADMIN_PASSWORD + cantaloupe: + secrets: + - TOMCAT_ADMIN_PASSWORD + drupal: + secrets: + - DB_ROOT_PASSWORD + - DRUPAL_DEFAULT_ACCOUNT_PASSWORD + - DRUPAL_DEFAULT_CONFIGDIR + - DRUPAL_DEFAULT_DB_PASSWORD + - DRUPAL_DEFAULT_SALT + - JWT_PRIVATE_KEY + - JWT_PUBLIC_KEY + fcrepo: + secrets: + - DB_ROOT_PASSWORD + - FCREPO_DB_PASSWORD + - JWT_ADMIN_TOKEN + - JWT_PUBLIC_KEY + homarus: + secrets: + - JWT_ADMIN_TOKEN + - JWT_PUBLIC_KEY + houdini: + secrets: + - JWT_ADMIN_TOKEN + - JWT_PUBLIC_KEY + hypercube: + secrets: + - JWT_ADMIN_TOKEN + - JWT_PUBLIC_KEY + mariadb: + secrets: + - DB_ROOT_PASSWORD + matomo: + secrets: + - DB_ROOT_PASSWORD + - MATOMO_DB_PASSWORD + - MATOMO_USER_PASS + milliner: + secrets: + - JWT_ADMIN_TOKEN + - JWT_PUBLIC_KEY + recast: + secrets: + - JWT_ADMIN_TOKEN + - JWT_PUBLIC_KEY diff --git a/docker-compose.solr.yml b/build/docker-compose/docker-compose.solr.yml similarity index 57% rename from docker-compose.solr.yml rename to build/docker-compose/docker-compose.solr.yml index f3981da5b..c9fa83559 100644 --- a/docker-compose.solr.yml +++ b/build/docker-compose/docker-compose.solr.yml @@ -3,26 +3,25 @@ networks: default: internal: true gateway: - external: - name: gateway + external: true services: solr: + restart: ${RESTART_POLICY:-unless-stopped} image: ${REPOSITORY:-islandora}/solr:${TAG:-latest} # No volumes are defined here, other docker-compose.yml files such as # docker-compose.demo.yml are expected to extend this definition based # on their workflow. networks: default: - # Allow services to use the edge name to reference this service. - # This allows links from the Drupal administration page to lead - # to the appropriate solr server. - aliases: - - solr-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - - solr-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} - gateway: # Do not expose in production. labels: # Do not expose in production. - - traefik.enable=true + - traefik.enable=${EXPOSE_SOLR:-false} - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-solr.loadbalancer.server.port=8983 - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-solr_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-solr - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-solr_http.entrypoints=http + - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-solr_http.entrypoints=solr + deploy: + resources: + limits: + memory: ${SOLR_MEMORY_LIMIT:-8G} + reservations: + memory: 6G diff --git a/build/docker-compose/docker-compose.starter.yml b/build/docker-compose/docker-compose.starter.yml new file mode 100644 index 000000000..f4f44103d --- /dev/null +++ b/build/docker-compose/docker-compose.starter.yml @@ -0,0 +1,36 @@ +# Container does not perform any initialization aside from importing env-vars from `confd`. +# Users are expected to manually set up their site using a combination of the following: +# - Makefile targets +# - composer requires / install +# - Drush commands +# - Manual changes to the codebase directory +version: "3.7" +networks: + default: + internal: true + gateway: + external: true +volumes: + drupal-sites-data: + solr-data: +services: + drupal: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/drupal:${TAG:-latest} + volumes: + - ../../codebase:/var/www/drupal:delegated + - drupal-sites-data:/var/www/drupal/web/sites/default/files + - solr-data:/opt/solr/server/solr + environment: + DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG: ${INSTALL_EXISTING_CONFIG} + DRUPAL_DEFAULT_PROFILE: ${DRUPAL_INSTALL_PROFILE} + depends_on: + # Requires a the very minimum a database. + - ${DRUPAL_DATABASE_SERVICE} + # Extends docker-compose.solr.yml + solr: + volumes: + # On a production site you may not want to take this approach but instead refer to each of the cores + # data directories specifically and maintain the configuration as part of a customized image, where + # in your configuration is Solr managed under source control somewhere. + - solr-data:/opt/solr/server/solr diff --git a/build/docker-compose/docker-compose.starter_dev.yml b/build/docker-compose/docker-compose.starter_dev.yml new file mode 100644 index 000000000..f4f44103d --- /dev/null +++ b/build/docker-compose/docker-compose.starter_dev.yml @@ -0,0 +1,36 @@ +# Container does not perform any initialization aside from importing env-vars from `confd`. +# Users are expected to manually set up their site using a combination of the following: +# - Makefile targets +# - composer requires / install +# - Drush commands +# - Manual changes to the codebase directory +version: "3.7" +networks: + default: + internal: true + gateway: + external: true +volumes: + drupal-sites-data: + solr-data: +services: + drupal: + restart: ${RESTART_POLICY:-unless-stopped} + image: ${REPOSITORY:-islandora}/drupal:${TAG:-latest} + volumes: + - ../../codebase:/var/www/drupal:delegated + - drupal-sites-data:/var/www/drupal/web/sites/default/files + - solr-data:/opt/solr/server/solr + environment: + DRUPAL_DEFAULT_INSTALL_EXISTING_CONFIG: ${INSTALL_EXISTING_CONFIG} + DRUPAL_DEFAULT_PROFILE: ${DRUPAL_INSTALL_PROFILE} + depends_on: + # Requires a the very minimum a database. + - ${DRUPAL_DATABASE_SERVICE} + # Extends docker-compose.solr.yml + solr: + volumes: + # On a production site you may not want to take this approach but instead refer to each of the cores + # data directories specifically and maintain the configuration as part of a customized image, where + # in your configuration is Solr managed under source control somewhere. + - solr-data:/opt/solr/server/solr diff --git a/build/docker-compose/docker-compose.traefik.yml b/build/docker-compose/docker-compose.traefik.yml new file mode 100644 index 000000000..d7a71b315 --- /dev/null +++ b/build/docker-compose/docker-compose.traefik.yml @@ -0,0 +1,74 @@ +# Allows traefik to work as the edge router for multiple projects. +# +# By default the rules will redirect to ${SERVICE}.${COMPOSE_PROJECT_NAME}.traefik.me +# if working on localhost and ${SERVICE}-${COMPOSE_PROJECT_NAME}-XX-XX-XX-XX.traefik.me +# if working remotely, where XX-XX-XX-XX is the dashed version of your IP address. +# +# For a traefik to be able to route traffic to a given container, that +# container needs to be on the `gateway` network, otherwise traefik will +# discover it via Docker but will not be able to redirect traffic to it. +version: "3.7" +networks: + gateway: + driver: bridge + internal: false +services: + traefik: + restart: ${RESTART_POLICY:-unless-stopped} + image: traefik:2.7 + container_name: traefik + # Do not set `api.insecure`, `api.dashboard`, `api.debug` to `true` in production. + # Also do not expose database 3306/5432, as an entry point. + # If the commands below are changed, please copy the changes to `docker-compose.acme.yml`. + command: + - --api.insecure=${EXPOSE_TRAEFIK_DASHBOARD:-false} + - --api.dashboard=${EXPOSE_TRAEFIK_DASHBOARD:-false} + - --api.debug=${EXPOSE_TRAEFIK_DASHBOARD:-false} + - --entryPoints.http.address=:80 + - --entryPoints.https.address=:443 + - --entryPoints.mysql.address=:3306 + - --entryPoints.postgresql.address=:5432 + - --entryPoints.fcrepo.address=:8081 + - --entryPoints.blazegraph.address=:8082 + - --entryPoints.activemq.address=:8161 + - --entryPoints.solr.address=:8983 + - --entryPoints.code-server.address=:8443 + - --log.level=${TRAEFIK_LOG_LEVEL:-ERROR} + - --providers.docker + - --providers.docker.network=gateway + - --providers.docker.exposedByDefault=false + - --providers.file.filename=/etc/traefik/tls.yml + - '--providers.docker.defaultRule=Host(`${DOMAIN}`)' + ports: + - 80:80 # drupal, cantaloupe, matomo + - 443:443 # https for ^^^ + - ${FEDORA_PORT:-8081}:8081 # fedora, needs to be exposed for cantaloupe + # Don't do any of these in production + - ${MYSQL_PORT:-3306}:3306 # mysql + - ${POSTGRES_PORT:-5432}:5432 # postgres + - ${TRAEFIK_DASHBOARD_PORT:-8080}:8080 # traefik admin dashboard - helpful for debugging + - ${BLAZEGRAPH_PORT:-8082}:8082 # blazegraph + - ${ACTIVEMQ_PORT:-8161}:8161 # activemq + - ${SOLR_PORT:-8983}:8983 # solr + - ${CODE_SERVER_PORT:-8443}:8443 # code-server + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ../../build/traefik-tls.yml:/etc/traefik/tls.yml + - ../../certs:/etc/ssl/traefik + labels: + # Do not expose in production. + - traefik.http.routers.api.service=api@internal + networks: + default: + aliases: + - ${DOMAIN} + gateway: {} + deploy: + resources: + limits: + memory: ${TRAEFIK_MEMORY_LIMIT:-8G} + reservations: + memory: 4G + drupal: + depends_on: + - traefik diff --git a/docker-compose.watchtower.yml b/build/docker-compose/docker-compose.watchtower.yml similarity index 57% rename from docker-compose.watchtower.yml rename to build/docker-compose/docker-compose.watchtower.yml index 8c5883d61..3c869ea97 100644 --- a/docker-compose.watchtower.yml +++ b/build/docker-compose/docker-compose.watchtower.yml @@ -4,7 +4,14 @@ version: "3.7" services: watchtower: - image: v2tec/watchtower + image: containrrr/watchtower + restart: ${RESTART_POLICY:-unless-stopped} volumes: - /var/run/docker.sock:/var/run/docker.sock command: --interval 1 --no-pull + deploy: + resources: + limits: + memory: ${WATCHTOWER_MEMORY_LIMIT:-2G} + reservations: + memory: 1G diff --git a/rootfs/etc/cont-init.d/04-custom-setup.sh b/build/rootfs/etc/cont-init.d/04-custom-setup.sh similarity index 79% rename from rootfs/etc/cont-init.d/04-custom-setup.sh rename to build/rootfs/etc/cont-init.d/04-custom-setup.sh index 11be60949..0b15ae944 100644 --- a/rootfs/etc/cont-init.d/04-custom-setup.sh +++ b/build/rootfs/etc/cont-init.d/04-custom-setup.sh @@ -9,7 +9,9 @@ function main { create_database "${site}" # Needs to be set to do an install from existing configuration. drush islandora:settings:create-settings-if-missing + local previous_owner_group=$(allow_settings_modifications ${site}) drush islandora:settings:set-config-sync-directory "${DRUPAL_DEFAULT_CONFIGDIR}" + restore_settings_ownership ${site} ${previous_owner_group} install_site "${site}" # Settings like the hash / flystem can be affected by environment variables at runtime. update_settings_php "${site}" @@ -22,7 +24,8 @@ function main { # to be up and running before they can complete. wait_for_required_services "${site}" # Create missing solr cores. - create_solr_core_with_default_config "${site}" + create_solr_core_with_default_config "${site}" || echo -e "\n\nERROR: SOLR was not initialized. Check the logs above for more details.\n\n" + # Create namespace assumed one per site. create_blazegraph_namespace_with_default_properties "${site}" # Need to run migration to get expected default content, now that our required services are running. @@ -33,9 +36,11 @@ function main { 'https://projects.iq.harvard.edu/fits']; -\$term->set('field_external_uri', \$default); -\$term->save(); +if (\$term) { + \$default = ['uri' => 'https://projects.iq.harvard.edu/fits']; + \$term->set('field_external_uri', \$default); + \$term->save(); +} EOF drush php:script /tmp/fix.php # Rebuild the cache. diff --git a/build/scripts/check-secrets.sh b/build/scripts/check-secrets.sh new file mode 100755 index 000000000..fd80c43bb --- /dev/null +++ b/build/scripts/check-secrets.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -e + +RED=$(tput -Txterm setaf 1) +GREEN=$(tput -Txterm setaf 2) +YELLOW=$(tput -Txterm setaf 3) +BLUE=$(tput -Txterm setaf 4) +RESET=$(tput -Txterm sgr0) +TARGET_MAX_CHAR_NUM=20 + +source .env || { + echo "${RED}ERROR: .env file not found.${RESET}" + exit 1 +} +FOUND_INSECURE_SECRETS=false + +function print_security_warning() { + if [ "${FOUND_INSECURE_SECRETS}" == true ]; then +cat << EOF + + + ${YELLOW} --- --- WARNING --- --- ${RESET}${RED} --- --- WARNING --- --- ${RESET} + + ${RED} + Using default values for secrets in a production environment is a + + Security Risk${RESET} + + Default values are identified in ${GREEN}$(pwd)/secrets/live/${RESET} + + If you are using the default values, you can either change the values of + the file found in $(pwd)/secrets/live/ + Or generate new secrets by running: + ${GREEN}make generate-secrets ${RESET} + + This will generate new secrets in /secrets/live/ but will not update + the ISLE containers. + + If you are not sure how to push updated secrets to ISLE, please consult + the documentation.${BLUE} + https://islandora.github.io/documentation/installation/docker-custom/#secrets + ${RESET} + + ${YELLOW} --- --- WARNING --- --- ${RESET}${RED} --- --- WARNING --- --- ${RESET} + + +EOF + fi +} + +function main() { + unameOut="$(uname -s)" + case "${unameOut}" in + Linux*) hash=sha1sum;; + *) hash="UNKNOWN" + esac + # Check if $USE_SECRETS is set to true. + if [ "$USE_SECRETS" = true ]; then + local secret_live=[]; + # Check if the $(pwd)/secrets/live directory is empty. + if [ "$(ls $(pwd)/secrets/live)" ]; then + local secret_live=($(find $(pwd)/secrets/live/* -exec basename {} \;)) + fi + fi + + local secret_templates=($(find $(pwd)/secrets/template/* -exec basename {} \;)) + + if [ ! "$(ls $(pwd)/secrets/live)" ]; then + echo -e "\n${YELLOW}Checking secrets...${RESET}" + echo " No secrets found in $(pwd)/secrets/live/" + echo "" + echo -e "Everything (Drupal, SQL, Solr, etc.) needs passwords to run. These passwords are stored in the ${GREEN}secrets${RESET} directory." + echo " 1. ${BLUE}Generate new secrets${RESET}" + echo " ╰ This will generate random new passwords and output them into $(pwd)/secrets/live/" + echo " 2. ${BLUE}Copy the existing secrets${RESET}" + echo " ╰ This will copy the existing secrets into the $(pwd)/secrets/live/. Every password will likely be 'password'." + echo " 3. ${BLUE}Create your own secrets${RESET}" + echo " ╰ This will copy the secrets from $(pwd)/secrets/template/ to $(pwd)/secrets/live/" + echo " and will exit so you can edit the secrets manually prior to building the Docker image." + echo " 4. ${BLUE}Exit${RESET}" + echo "" + read -p "Enter your choice: " choice + case $choice in + 1) + echo "Generating new secrets" + docker run --rm -t \ + -v $(pwd)/secrets:/secrets \ + -v $(pwd)/build/scripts/generate-secrets.sh:/generate-secrets.sh \ + -w / \ + --entrypoint bash \ + ${REPOSITORY}/drupal:${TAG} -c "/generate-secrets.sh && chown -R `id -u`:`id -g` /secrets" + echo -e "\n${GREEN}Secrets generated.${RESET}" + ;; + 2) + echo "Copying existing secrets" + cp -n $(pwd)/secrets/template/* $(pwd)/secrets/live/ + echo -e "\n${GREEN}Secrets copied.${RESET}" + ;; + 3) + echo "Creating your own secrets. Copying templates to live directory." + cp -n $(pwd)/secrets/template/* $(pwd)/secrets/live/ + echo -e "\nTemplate ${GREEN}Secrets copied to $(pwd)/secrets/live/.${RESET}" + echo "You can now edit the secrets in $(pwd)/secrets/live/ and then run: ${GREEN}make up${RESET} to start building the Docker image." + exit 1 + ;; + *) + echo "Exiting" + exit 1 + ;; + esac + fi + + local secret_live=($(find $(pwd)/secrets/live/* -exec basename {} \;)) + for secret in "${secret_templates[@]}"; do + if [[ ! "${secret_live[@]}" =~ "${secret}" ]]; then + missing_secret_identified=true + break; + fi + + if [[ $hash == "UNKNOWN" ]]; then + if [[ $(cat secrets/template/${secret}) == $(cat secrets/live/${secret}) ]]; then + # Ignore the config location directory. This won't pose a security risk. + if [[ ! "${secret}" = "DRUPAL_DEFAULT_CONFIGDIR" ]]; then + echo -e "${RED}Default Secret${RESET} ${BLUE}->${RESET} $(pwd)/secrets/live/${secret}" + FOUND_INSECURE_SECRETS=true + fi + fi + else + if [[ "$($hash $(pwd)/secrets/template/${secret}| awk '{print $1}')" == "$($hash $(pwd)/secrets/live/${secret}| awk '{print $1}')" ]]; then + # Ignore the config location directory. This won't pose a security risk. + if [[ ! "${secret}" = "DRUPAL_DEFAULT_CONFIGDIR" ]]; then + echo -e "${RED}Default Secret${RESET} ${BLUE}->${RESET} $(pwd)/secrets/live/${secret}" + FOUND_INSECURE_SECRETS=true + fi + fi + fi + done + + if [ "${missing_secret_identified}" = true ]; then + echo -e "\n\nIdentified a few missing SECRETS.\n" + echo -e " Would you like to copy the missing secrets from $(pwd)/secrets/template/? [y/N] " + read thr_ans + if [[ ${thr_ans} == [yY] ]] ; then + echo "" + for secret in "${secret_templates[@]}"; do + if [[ ! "${secret_live[@]}" =~ "${secret}" ]]; then + echo "MISSING: $(pwd)/secrets/live/${secret}" + echo -e " Copying ${RED}${secret}${RESET} to $(pwd)/secrets/live/${GREEN}${secret}${RESET}\n" + cp -n $(pwd)/secrets/template/${secret} $(pwd)/secrets/live/${secret} + echo "" + fi + done + else + echo -e "\nPlease update the missing secrets before continuing.\n\n" + exit 1 + fi + fi + + # Check if Salt matches the one in secrets/live/. + SALT=$(echo $(docker-compose exec drupal with-contenv bash -lc "cat web/sites/default/settings.php | grep hash_salt | grep '^\$settings' | cut -d\= -f2| cut -d\' -f2 | cut -f1 -d\"'\" | tr -d '\n' | cut -f1 -d\"%\"")) + SETTINGS_SALT=$(echo $(cat secrets/live/DRUPAL_DEFAULT_SALT | tr -d '\n' | cut -f1 -d"%")) + if [[ $(echo "${SALT}") != $(echo "${SETTINGS_SALT}") ]]; then + echo "${SALT} ${SETTINGS_SALT} Updates to the salt are not automatically added to web/sites/default/settings.php file. Please make this change manually and then run the same ${BLUE}make down && make up${RESET} command again." + fi +} + +# Just incase the wishes to automate generation of secrets. +if [[ $1 == 'yes' ]]; then + docker run --rm -t \ + -v $(pwd)/secrets:/secrets \ + -v $(pwd)/build/scripts/generate-secrets.sh:/generate-secrets.sh \ + -w / \ + --entrypoint bash \ + ${REPOSITORY}/drupal:${TAG} -c "/generate-secrets.sh && chown -R `id -u`:`id -g` /secrets" + echo -e "\n${GREEN}Secrets generated.${RESET}" +fi + +main +print_security_warning +echo -e "\nCheck secrets is ${GREEN}done${RESET}.\n\n" \ No newline at end of file diff --git a/build/scripts/generate-secrets.sh b/build/scripts/generate-secrets.sh new file mode 100755 index 000000000..56aa48026 --- /dev/null +++ b/build/scripts/generate-secrets.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -e + +# This script is meant to only be called via the Makefile not independently. + +function generate_jwt_keys() { + openssl genrsa -out /tmp/private.key 2048 &>/dev/null + openssl rsa -pubout -in /tmp/private.key -out /tmp/public.key &>/dev/null +} + +function generate_matomo_password() { + # Password is in two parts, the part that is human readable and entered into + # the form, and the hashed version which is written into the database. + random_secret 'A-Za-z0-9' 48 MATOMO_USER_PASS_NON_HASHED + MATOMO_USER_PASS=$(cat /secrets/live/MATOMO_USER_PASS_NON_HASHED) + php -r "echo password_hash(md5('${MATOMO_USER_PASS}'), PASSWORD_DEFAULT);" >/secrets/live/MATOMO_USER_PASS +} + +function random_secret() { + local characters=${1} + local size=${2} + local name=${3} + tr -dc "${characters}" /secrets/live/"${name}" +} + +function main() { + echo "Generating Secrets" + local secret_templates=($(find ../secrets/template/* -exec basename {} \;)) + generate_jwt_keys + for secret in "${secret_templates[@]}"; do + case "${secret}" in + DRUPAL_DEFAULT_CONFIGDIR) + cp /secrets/template/DRUPAL_DEFAULT_CONFIGDIR /secrets/live/DRUPAL_DEFAULT_CONFIGDIR + ;; + DRUPAL_DEFAULT_SALT) + random_secret 'A-Za-z0-9-_' 74 DRUPAL_DEFAULT_SALT + ;; + JWT_ADMIN_TOKEN) + random_secret 'A-Za-z0-9' 64 JWT_ADMIN_TOKEN + ;; + JWT_PRIVATE_KEY) + cp /tmp/private.key /secrets/live/JWT_PRIVATE_KEY + ;; + JWT_PUBLIC_KEY) + cp /tmp/public.key /secrets/live/JWT_PUBLIC_KEY + ;; + MATOMO_USER_PASS) + generate_matomo_password + ;; + *) + random_secret 'A-Za-z0-9' 48 "${secret}" + ;; + esac + echo "Generated or copied ${secret}!" + done + # Make sure they are only readable by their owner. + chmod 600 /secrets/live/* +} +main diff --git a/tls.yml b/build/traefik-tls.yml similarity index 100% rename from tls.yml rename to build/traefik-tls.yml diff --git a/custom.Makefile b/custom.Makefile new file mode 100644 index 000000000..57c0a7512 --- /dev/null +++ b/custom.Makefile @@ -0,0 +1,341 @@ +# To write custom Makefile commands and have them show up under `make help`. +# +# .PHONY: function_name +# .SILENT: function_name +# ## JHU: Updates codebase folder to be owned by the host user and nginx group. +# function_name: +# ⟼ Tab (not space characters) and each line is executed as part of this function. +# +DOCKCOMPOSE_FILE = $(CURDIR)/docker-compose.yml +ifneq ("$(wildcard $(DOCKCOMPOSE_FILE))","") + DF_FILE_EXISTS = 1 +else + DF_FILE_EXISTS = 0 +endif + +ifneq ("$(wildcard /etc/server-type.conf)","") + SERVER_TYPE = $(shell cat /etc/server-type.conf) +else + SERVER_TYPE = local +endif + +.PHONY: wait-for-endpoint +.SILENT: wait-for-endpoint +wait-for-endpoint: + @echo "Checking if the https://$(DOMAIN)/ endpoint is available..." + @while ! curl -k -s -o /dev/null -w "%{http_code}" https://$(DOMAIN)/ | grep -q "200"; do \ + echo "Waiting for https://$(DOMAIN)/ endpoint to be available..."; \ + sleep 5; \ + done + @echo "Endpoint is available!" + +.PHONY: jhu_check_and_warn +.SILENT: jhu_check_and_warn +jhu_check_and_warn: + if [ $(SERVER_TYPE) != 'local' ]; then \ + echo "Not allowed on the $(SERVER_TYPE) environment"; \ + exit 1; \ + fi + +.PHONY: jhu_generate-secrets +.SILENT: jhu_generate-secrets +jhu_generate-secrets: QUOTED_CURDIR = "$(CURDIR)" +jhu_generate-secrets: + @echo "" + cp -r secrets/template/* secrets/live + $(MAKE) generate-secrets + @echo " jhu_generate-secrets └─ Done" + @echo "" + +.PHONY: set-codebase-owner +.SILENT: set-codebase-owner +## JHU: Updates codebase folder to be owned by the host user and nginx group. +set-codebase-owner: + @echo "" + @echo "Setting codebase/ folder owner back to $(shell id -u):101" + if [ -n "$$(docker ps -q -f name=drupal)" ]; then \ + echo " └─ Using docker-compose codebase/ directory"; \ + docker-compose exec -T drupal with-contenv bash -lc "find . -not -user $(shell id -u) -not -path '*/sites/default/files' -exec chown $(shell id -u):101 {} \;" ; \ + docker-compose exec -T drupal with-contenv bash -lc "find . -not -group 101 -not -path '*/sites/default/files' -exec chown $(shell id -u):101 {} \;" ; \ + elif [ -d "codebase" ]; then \ + echo " └─ Using local codebase/ directory"; \ + sudo find ./codebase -not -user $(shell id -u) -not -path '*/sites/default/files' -exec chown $(shell id -u):101 {} \; ; \ + sudo find ./codebase -not -group 101 -not -path '*/sites/default/files' -exec chown $(shell id -u):101 {} \; ; \ + else \ + echo " └─ No codebase/ directory found, skipping"; \ + fi + @echo " └─ Done" + @echo "" + +.PHONY: jhu_solr +.SILENT: jhu_solr +## JHU: This pulls the Solr config from Drupal and puts it in the Solr container. +jhu_solr: + @echo "" + @echo "Installing missing field types" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api-solr:install-missing-fieldtypes" + # docker-compose exec -T drupal bash -c '/bin/rm -f /opt/solr/server/solr/ISLANDORA/conf/solrconfig_extra.xml ; /bin/cp -f web/modules/contrib/search_api_solr/jump-start/solr7/config-set/solrconfig_extra.xml /opt/solr/server/solr/ISLANDORA/conf/solrconfig_extra.xml' + @echo "Removing solrconfig_extra.xml" + docker-compose exec -T drupal bash -c '/bin/rm -rf /opt/solr/server/solr/ISLANDORA/conf/' + @echo "Pulling Solr config from Drupal" + -docker-compose exec -T drupal with-contenv bash -lc "touch /var/www/drupal/solrconfig.zip && chown nginx: /var/www/drupal/solrconfig.zip" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api-solr:get-server-config default_solr_server /var/www/drupal/solrconfig.zip" + docker-compose exec -T drupal with-contenv bash -lc "unzip /var/www/drupal/solrconfig.zip -d /opt/solr/server/solr/ISLANDORA/conf/ -o" + docker-compose exec -T drupal with-contenv bash -lc "rm -f /var/www/drupal/solrconfig.zip" + @echo "Restarting solr" + docker-compose restart solr + # Check if Solr is up + @echo "Checking if Solr's healthy" + sleep 5 + docker-compose exec -T solr bash -c 'curl -s http://localhost:8983/solr/admin/info/system?wt=json' | jq -r .lucene || (echo "Solr is not healthy, waiting 10 seconds." && sleep 10) + docker-compose exec -T drupal with-contenv bash -lc "drush cr" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api:clear" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api:disable-all" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api:enable-all" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api-solr:finalize-index --force" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api-reindex" + docker-compose exec -T drupal with-contenv bash -lc "drush search-api-index" + @echo " └─ Done" + +.PHONY: jhu_up_without_rebuilding +## JHU: Make a local site with codebase directory bind mounted, using cloned starter site but without rebuilding the build process. +jhu_up_without_rebuilding: + @echo "" + if [ $(DF_FILE_EXISTS) -eq 0 ]; then \ + echo "docker-compose.yml does not exist, creating starter site"; \ + fi + docker-compose up -d --build + $(MAKE) set-codebase-owner + $(MAKE) jhu_config_import + @echo " └─ Done" + +.PHONY: jhu_up +## JHU: Make a local site with codebase directory bind mounted, using cloned starter site. +jhu_up: QUOTED_CURDIR = "$(CURDIR)" +jhu_up: jhu_generate-secrets + @echo "" + if [ $(DF_FILE_EXISTS) -eq 1 ]; then \ + echo "docker-compose.yml already exists, skipping starter site creation"; \ + docker-compose up -d --remove-orphans ; \ + echo " └─ Done"; \ + echo ""; \ + echo " Forcing an exit to prevent running creation steps again."; \ + echo ""; \ + fi + @echo "Running Starter-init" + $(MAKE) starter-init ENVIRONMENT=starter_dev + if [ -z "$$(ls -A $(QUOTED_CURDIR)/codebase)" ]; then \ + echo "codebase/ directory is empty, cloning it"; \ + docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone -b main https://github.com/jhu-idc/idc-codebase /home/root;'; \ + echo "codebase/ was cloned"; \ + fi + @echo "Wait for the /var/www/drupal/composer.json file to be available" + while ! docker compose exec -T drupal with-contenv bash -lc 'test -f /var/www/drupal/composer.json'; do \ + echo "Waiting for /var/www/drupal/composer.json file to be available..."; \ + sleep 2; \ + done + $(MAKE) set-codebase-owner + docker-compose up -d --remove-orphans + # The rest of this should be moved into another function. + -docker-compose exec -T drupal with-contenv bash -lc 'rm -rf vendor/ web/modules/contrib/* web/themes/contrib/*' + docker compose exec -T drupal with-contenv bash -lc 'chown -R nginx:nginx /var/www/drupal/ && su nginx -s /bin/bash -c "composer install"' + -docker-compose exec -T drupal with-contenv bash -lc 'git config --global --add safe.directory /var/www/drupal/web/modules/contrib/islandora_fits' + $(MAKE) set-codebase-owner + $(MAKE) drupal-database update-settings-php + docker-compose exec -T drupal with-contenv bash -lc "drush si -y --existing-config minimal --account-pass $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)" + docker-compose exec -T drupal with-contenv bash -lc "drush -l $(SITE) user:role:add fedoraadmin admin" + MIGRATE_IMPORT_USER_OPTION=--userid=1 $(MAKE) hydrate + docker-compose exec -T drupal with-contenv bash -lc 'drush -l $(SITE) migrate:import --userid=1 islandora_fits_tags' + $(MAKE) jhu_config_import + docker-compose exec -T drupal with-contenv bash -lc 'composer require drupal/migrate_tools ; drush pm:enable -y migrate_tools,idc_default_migration && drush migrate:import idc_default_migration_menu_link_main' + $(MAKE) jhu_solr + docker-compose exec -T drupal with-contenv bash -lc 'mkdir -p web/sites/default/files/styles/thumbnail/public/media-icons/generic' + docker-compose exec -T drupal with-contenv bash -lc 'cp web/core/modules/media/images/icons/* web/sites/default/files/media-icons/generic/' + docker-compose exec -T drupal with-contenv bash -lc 'cp web/core/modules/media/images/icons/generic.png web/sites/default/files/media-icons/generic' + docker-compose exec -T drupal with-contenv bash -lc 'chown nginx: web/sites/default/files/media-icons/generic/generic.png' + docker-compose exec -T drupal with-contenv bash -lc 'mkdir -p /var/www/drupal/private ; chown -R nginx:nginx /var/www/drupal/private ; chmod -R 755 /var/www/drupal/private' + sudo rsync -avz scripts/services.yml codebase/web/sites/default/services.yml + sudo rsync -avz scripts/default.services.yml codebase/web/sites/default/default.services.yml + $(MAKE) wait-for-endpoint + curl -k -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@build/demo-data/jhu_homepage.json" https://${DOMAIN}/node?_format=json + curl -k -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@build/demo-data/browse-collections.json" https://${DOMAIN}/node?_format=json + docker-compose down + docker-compose up -d + # drush config:set system.logging error_level verbose -y + +.PHONY: jhu_demo_content +.SILENT: jhu_demo_content +## JHU: Helper function for demo sites: do a workbench import of sample objects +jhu_demo_content: QUOTED_CURDIR = "$(CURDIR)" +jhu_demo_content: + # fetch repo that has csv and binaries to data/samples + # if prod do this by default + [ -f "secrets/live/DRUPAL_DEFAULT_ADMIN_USERNAME" ] || (echo "admin" > secrets/live/DRUPAL_DEFAULT_ADMIN_USERNAME) + [ -f "secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD" ] || (echo "password" > secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) + -docker-compose exec -T drupal with-contenv bash -lc "composer require mjordan/islandora_workbench_integration" + -docker-compose exec -T drupal with-contenv bash -lc "drush en -y islandora_workbench_integration" + [ -d "islandora_workbench" ] || (git clone https://github.com/mjordan/islandora_workbench) + [ -d "islandora_workbench/islandora_workbench_demo_content" ] || (git clone https://github.com/DonRichards/islandora_workbench_demo_content islandora_workbench/islandora_workbench_demo_content) + # $(SED_DASH_I) 's/^nopassword.*/password\: $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) /g' islandora_workbench/islandora_workbench_demo_content/example_content.yml + # Hack to disable SSL verification/warnings | InsecureRequestWarning: Unverified HTTPS request is being made to host 'islandora.traefik.me'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings + FILE="islandora_workbench/workbench"; \ + if ! grep -q "import requests" "$$FILE"; then \ + echo "Fixing the file: $$FILE"; \ + sed -i '/def create():/i import requests\nfrom requests.packages.urllib3.exceptions import InsecureRequestWarning\n\n# Disable SSL/TLS related warnings\nrequests.packages.urllib3.disable_warnings(InsecureRequestWarning)' "$$FILE"; \ + fi + # Set the Username within the YAML files. + find islandora_workbench/islandora_workbench_demo_content/ -type f -name '*.yml' -exec $(SED_DASH_I) 's/^username.*/username\: $(shell cat secrets/live/DRUPAL_DEFAULT_ADMIN_USERNAME)/g' {} + + # Set the Password within the YAML files. + find islandora_workbench/islandora_workbench_demo_content/ -type f -name '*.yml' -exec $(SED_DASH_I) 's/^nopassword.*/password\: $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)/g' {} + + # Set the Domain within the YAML files. + find islandora_workbench/islandora_workbench_demo_content/ -type f -name '*.yml' -exec $(SED_DASH_I) '/^host:/s/.*/host: "$(subst /,\/,$(subst .,\.,$(SITE)))\/"/' {} + + # Look up the Node ID of the collection you want to import into and replace the value with one live on the site. + # This is a hack to work around for workbench. There is a better method. + @echo "Starting hack for workbench." + bash temp_migrate_fix.sh + @echo "Build the workbench docker image." + cd islandora_workbench && DOCKER_BUILDKIT=0 docker build -t workbench-docker . + @echo "Migrate/Import taxonomy terms." + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_geo_location.yml" + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_copyright_and_use.yml" + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_family.yml" + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_person.yml" + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_corporate_body.yml" + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_genre.yml" + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/idc_subject.yml" + @echo "Migrate/Import taxonomy terms complete." + @echo "Migrate/Import collections and objects." + cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/jhu_root_collections.yml" + cd /home/don/github/idc-isle-dc/islandora_workbench/islandora_workbench_demo_content/migration_from_production/ ; head -n 101 islandora_objects.csv > islandora_objects_first_100.csv ; cd /home/don/github/idc-isle-dc + cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/islandora_workbench_demo_content/islandora_objects.yml" + @echo "Migrate/Import collections and objects complete." + # Example of a rollback + # cd islandora_workbench && docker run -it --rm --network="host" -v .:/workbench/ --name my-running-workbench workbench-docker bash -lc "./workbench --config /workbench/rollback.yml" + + $(MAKE) jhu_solr + @echo " └─ Done" + +.PHONY: jhu_clean +.SILENT: jhu_clean +## JHU: Destroys all local data, including codebase, docker volumes, and untracked/ignored files. +jhu_clean: + @echo "**DANGER** About to rm your SERVER data subdirs, your docker volumes, islandora_workbench, certs, secrets, codebase/, and all untracked/ignored files (including changes to .env)." + $(MAKE) confirm + docker-compose down -v --remove-orphans || true + # sudo rm -fr certs secrets/live/* docker-compose.yml codebase islandora_workbench + # -git stash + # -git clean -xffd . + # -git checkout . + @echo "Codebase/ was completely removed." + @echo " └─ Done" + +.PHONY: jhu_reset +.SILENT: jhu_reset +## JHU: Destroys all local data, docker volumes, without removing codebase or workbench. +jhu_reset: + @echo "**DANGER** About to rm your SERVER data subdirs, your docker volumes, certs, secrets." + $(MAKE) confirm + docker-compose down -v --remove-orphans || true + sudo rm -fr certs secrets/live/* docker-compose.yml + @echo " └─ Done shutting down containers and removing volumes." + $(MAKE) jhu_up + +.PHONY: jhu_down +.SILENT: jhu_down +## JHU: Brings the local site down without destroying data. +jhu_down: + -docker-compose down + +.PHONY: jhu_config_export +.SILENT: jhu_config_export +## JHU: Exports the sites configuration. +jhu_config_export: + cd codebase && rm -rf config/sync/* && git checkout -- config/sync + docker-compose exec drupal with-contenv bash -lc "chown -R nginx: /var/www/drupal/config/sync/" + docker-compose exec -T drupal drush -l $(SITE) config:export -y + $(MAKE) set-codebase-owner + +.PHONY: jhu_config_import +.SILENT: jhu_config_import +## JHU: Imports the sites configuration. +jhu_config_import: + $(MAKE) set-codebase-owner + @echo "Wait for the /var/www/drupal directory to be available" + while ! docker compose exec -T drupal with-contenv bash -lc 'test -d /var/www/drupal'; do \ + echo "Waiting for /var/www/drupal directory to be available..."; \ + sleep 2; \ + done + docker compose exec -T drupal with-contenv bash -lc 'chown -R nginx:nginx /var/www/drupal/ && su nginx -s /bin/bash -c "composer install"' + docker-compose exec drupal with-contenv bash -lc "chown -R nginx: /var/www/drupal/config/sync/" + docker-compose exec -T drupal drush -l $(SITE) config:import -y --debug + $(MAKE) set-codebase-owner + +.PHONY: jhu_dev_tools_enable +.SILENT: jhu_dev_tools_enable +## JHU: Enables devel and devel_generate modules. +jhu_dev_tools_enable: + $(MAKE) set-codebase-owner + docker-compose exec drupal with-contenv bash -lc "git config --global --add safe.directory /var/www/drupal/vendor/drupal/coder" + -docker-compose exec drupal with-contenv bash -lc "cp ~/.bashrc ~/.bashrc_BAK || echo 'No .bashrc file to backup'" + docker-compose exec drupal with-contenv bash -lc "echo \"alias drupal='vendor/drupal/console/bin/drupal'\" > ~/.bashrc" + docker-compose exec drupal with-contenv bash -lc "echo \"alias phpcs='vendor/squizlabs/php_codesniffer/bin/phpcs'\" >> ~/.bashrc" + docker-compose exec drupal with-contenv bash -lc "echo \"alias phpcbf='vendor/squizlabs/php_codesniffer/bin/phpcbf'\" >> ~/.bashrc" + docker-compose exec drupal with-contenv bash -lc "composer require drupal/coder --dev && git config --global --add safe.directory /var/www/drupal/vendor/drupal/coder" + docker-compose exec drupal with-contenv bash -lc "vendor/squizlabs/php_codesniffer/bin/phpcs --config-set installed_paths vendor/slevomat/coding-standard vendor/phpcompatibility/php-compatibility vendor/drupal/coder/coder_sniffer && vendor/squizlabs/php_codesniffer/bin/phpcs --config-set default_standard Drupal" + if [ ! -f "codebase/web/sites/default/services.yml" ]; then cp scripts/services.yml codebase/web/sites/default/services.yml; fi + docker-compose exec drupal with-contenv bash -lc "drush en devel -y && drush cr" + +.PHONY: jhu_repos_export +.SILENT: jhu_repos_export +## JHU: This copies the codebase directory and theme directory to a parent directory. +jhu_repos_export: + $(MAKE) jhu_config_export + -sudo rsync -avz --exclude '.git' --exclude '.gitignore' --exclude '.github' codebase/ ../idc-codebase --delete + -sudo chown -R $(USER): ../idc-codebase + -sudo rsync -avz --exclude '.git' --exclude '.gitignore' --exclude '.github' codebase/web/themes/contrib/idc_ui_theme_boots ../ --delete + -sudo rsync -avz --exclude '.git' --exclude '.gitignore' --exclude '.github' codebase/web/modules/contrib/idc_default_migration ../ --delete + -sudo rsync -avz --exclude '.git' --exclude '.gitignore' --exclude '.github' islandora_workbench/islandora_workbench_demo_content ../ --delete + $(MAKE) set-codebase-owner + +.PHONY: jhu_sync_repos +.SILENT: jhu_sync_repos +## JHU: This copies the codebase repo and the theme directory from the parent directory. +jhu_sync_repos: + $(MAKE) set-codebase-owner + [ -d "../idc-codebase/" ] && rsync -avz ../idc-codebase/ codebase + [ -d "../idc_ui_theme_boots/" ] && rsync -avz ../idc_ui_theme_boots/ codebase/web/themes/contrib/idc_ui_theme_boots --delete + [ -d "../idc_default_migration/" ] && rsync -avz ../idc_default_migration codebase/web/modules/contrib/idc_default_migration --delete + mkdir -p islandora_workbench/islandora_workbench_demo_content + [ -d "../islandora_workbench_demo_content/" ] && rsync -avz islandora_workbench/islandora_workbench_demo_content --delete + docker-compose exec drupal with-contenv bash -lc 'cp -r /var/www/drupal/web/core/assets/vendor/popperjs /var/www/drupal/web/libraries/popperjs ; cp -r /var/www/drupal/vendor/twbs/bootstrap /var/www/drupal/web/libraries/bootstrap ; chown -R nginx: /var/www/drupal/web/libraries/' + docker-compose exec drupal with-contenv bash -lc 'find /var/www/drupal/web/libraries/ -type d -exec chmod 755 {} \; -o -type f -exec chmod 644 {} \;' + $(MAKE) set-codebase-owner + +.PHONY: test +.SILENT: test +## JHU: This test should import the demo content. +test: + $(MAKE) jhu_demo_content + @echo " └─ Done" + +.PHONY: jhu_update_theme +.SILENT: jhu_update_theme +## JHU: This updates the theme's hash in composer. +jhu_update_theme: + rm -rf codebase/web/themes/contrib/idc_ui_theme_boots + docker-compose exec drupal with-contenv bash -lc 'composer clearcache && composer update --prefer-source "islandora/idc_ui_theme_boots" --with-all-dependencies' + +.PHONY: jhu_change_domain +.SILENT: jhu_change_domain +jhu_change_domain: + # Change the DOMAIN= in .env + sed -i "s/DOMAIN=.*/DOMAIN=$(DOMAIN)/g" .env + +.PHONY: jhu_stop +.SILENT: jhu_stop +## JHU: Turn off idc without losing anything. +jhu_stop: + @echo "" + @echo "Stopping containers" + docker-compose stop + @echo " └─ Done" diff --git a/docker-compose.alpaca.yml b/docker-compose.alpaca.yml deleted file mode 100644 index 90b3bc020..000000000 --- a/docker-compose.alpaca.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: "3.7" -networks: - default: - internal: true -services: - alpaca: - image: ${REPOSITORY:-islandora}/alpaca:${TAG:-latest} diff --git a/docker-compose.crayfish.yml b/docker-compose.crayfish.yml deleted file mode 100644 index 30b6df1b9..000000000 --- a/docker-compose.crayfish.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.7" -networks: - default: - internal: true -services: - gemini: - image: ${REPOSITORY:-islandora}/gemini:${TAG:-latest} - homarus: - image: ${REPOSITORY:-islandora}/homarus:${TAG:-latest} - houdini: - image: ${REPOSITORY:-islandora}/houdini:${TAG:-latest} - hypercube: - image: ${REPOSITORY:-islandora}/hypercube:${TAG:-latest} - milliner: - image: ${REPOSITORY:-islandora}/milliner:${TAG:-latest} - recast: - image: ${REPOSITORY:-islandora}/recast:${TAG:-latest} diff --git a/docker-compose.crayfits.yml b/docker-compose.crayfits.yml deleted file mode 100644 index 6afac5778..000000000 --- a/docker-compose.crayfits.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: "3.7" -networks: - default: - internal: true -services: - crayfits: - image: ${REPOSITORY:-islandora}/crayfits:${TAG:-latest} - depends_on: - - fits - fits: - image: ${REPOSITORY:-islandora}/fits:${TAG:-latest} diff --git a/docker-compose.drupal.mariadb.yml b/docker-compose.drupal.mariadb.yml deleted file mode 100644 index 5e2d2bf21..000000000 --- a/docker-compose.drupal.mariadb.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3.7" -services: - # Override defaults so Mariadb is used as the database for this service. - drupal: - environment: - DRUPAL_DEFAULT_DB_DRIVER: mysql - # Use the edge name so the Drush on the host machine can access it. - DRUPAL_DEFAULT_DB_HOST: mariadb-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - DRUPAL_DEFAULT_DB_PORT: 3306 diff --git a/docker-compose.drupal.postgresql.yml b/docker-compose.drupal.postgresql.yml deleted file mode 100644 index 118b44842..000000000 --- a/docker-compose.drupal.postgresql.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3.7" -services: - # Override defaults so PostgreSQL is used as the database for this service. - drupal: - environment: - DRUPAL_DEFAULT_DB_DRIVER: pgsql - # Use the edge name so the Drush on the host machine can access it. - DRUPAL_DEFAULT_DB_HOST: postgresql-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - DRUPAL_DEFAULT_DB_PORT: 5432 diff --git a/docker-compose.fcrepo.yml b/docker-compose.fcrepo.yml deleted file mode 100644 index 3f5d0717b..000000000 --- a/docker-compose.fcrepo.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: "3.7" -networks: - default: - internal: true - gateway: - external: - name: gateway -volumes: - fcrepo-data: -services: - fcrepo: - image: ${REPOSITORY:-islandora}/fcrepo:${TAG:-latest} - environment: - FCREPO_ALLOW_EXTERNAL_DRUPAL: http://islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}/ - volumes: - - fcrepo-data:/data - depends_on: - - activemq - networks: - default: - aliases: - # Flysystem allows us to specify a single URL for Fedora, which is - # used for display as well. This means this service edge name must - # also be used in the default network for access by the `drupal` - # service. - - fcrepo-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - - fcrepo-${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} - gateway: # May want to disable in production if you wish to hide Fedora REST API from the internet. - labels: - # Do not expose in production. - - traefik.enable=true - - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo.loadbalancer.server.port=80 - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo_http.service=${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo - - traefik.http.routers.${COMPOSE_PROJECT_NAME-isle-dc}-fcrepo_http.entrypoints=http diff --git a/docker-compose.sample.env.yml b/docker-compose.sample.env.yml deleted file mode 100644 index a1cb1a383..000000000 --- a/docker-compose.sample.env.yml +++ /dev/null @@ -1,321 +0,0 @@ -# This is just an example of which environment variables you can override for -# each of the services. If docker-compose.env.yml does not exist this file will -# be used to create it. -# -# These are, at a bare minimum, the settings you need to configure to deploy -# between environments. This is not an exhaustive list of all the environment -# variables available for each service. For the full list see the respective -# README.md file for the service in the isle-buildkit repository: -# -# - activemq: https://github.com/Islandora-Devops/isle-buildkit/tree/main/activemq/README.md -# - alpaca: https://github.com/Islandora-Devops/isle-buildkit/tree/main/alpaca/README.md -# - blazegraph: https://github.com/Islandora-Devops/isle-buildkit/tree/main/blazegraph/README.md -# - cantaloupe: https://github.com/Islandora-Devops/isle-buildkit/tree/main/cantaloupe/README.md -# - crayfits: https://github.com/Islandora-Devops/isle-buildkit/tree/main/crayfits/README.md -# - drupal: https://github.com/Islandora-Devops/isle-buildkit/tree/main/drupal/README.md -# - fcrepo: https://github.com/Islandora-Devops/isle-buildkit/tree/main/fcrepo/README.md -# - fits: https://github.com/Islandora-Devops/isle-buildkit/tree/main/fits/README.md -# - gemini: https://github.com/Islandora-Devops/isle-buildkit/tree/main/gemini/README.md -# - homarus: https://github.com/Islandora-Devops/isle-buildkit/tree/main/homarus/README.md -# - houdini: https://github.com/Islandora-Devops/isle-buildkit/tree/main/houdini/README.md -# - hypercube: https://github.com/Islandora-Devops/isle-buildkit/tree/main/hypercube/README.md -# - mariadb: https://github.com/Islandora-Devops/isle-buildkit/tree/main/mariadb/README.md -# - matomo: https://github.com/Islandora-Devops/isle-buildkit/tree/main/matomo/README.md -# - milliner: https://github.com/Islandora-Devops/isle-buildkit/tree/main/milliner/README.md -# - recast: https://github.com/Islandora-Devops/isle-buildkit/tree/main/recast/README.md -# - solr: https://github.com/Islandora-Devops/isle-buildkit/tree/main/solr/README.md -# -# Unfortunately there isn't a way to a way to use a variable value in multiple -# places, except through the `.env` file, which does not support multi-line -# variables, so for consistency sake all the values are duplicated here, rather -# than doing a mixed approach where some are defined in `env` files and some here. -version: "3.7" -services: - activemq: - environment: - # - # Password for default JAAS authentication: https://activemq.apache.org/security - # - ACTIVEMQ_PASSWORD: password - # - # Password to log into the ActiveMQ Web Console: https://activemq.apache.org/web-console - # - ACTIVEMQ_WEB_ADMIN_PASSWORD: password - alpaca: - environment: - # - # Must be exactly the same as ACTIVEMQ_PASSWORD. - # - ALPACA_ACTIVEMQ_PASSWORD: password - # - # Password for admin user: https://karaf.apache.org/manual/latest/security - # - ALPACA_KARAF_ADMIN_PASSWORD: password - blazegraph: - # No environment variables require overriding. - cantaloupe: - # No environment variables require overriding. - crayfits: - # No environment variables require overriding. - mariadb: - environment: - # - # Password for the root user. - # - MYSQL_ROOT_PASSWORD: password - # If you use PostgreSQL for any services uncomment this section. - # postgresql: - # environment: - # # - # # Password for the root user. - # # - # POSTGRESQL_ROOT_PASSWORD: password - drupal: - environment: - # - # Settings which the Drupal site exposes as links need to be corrected to use the edge name for those services. - # If creating sub-sites you must specify each of these as the defaults would cause odd issues where it pulls - # content from the main site rather than the sub site. - # - # If you are using an IP address with traefik.me, change the dot between COMPOSE_PROJECT_NAME and - # DRUPAL_SITE_HOST to a hyphen (i.e. ${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - # becomes ${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} - # - DRUPAL_DEFAULT_CANTALOUPE_URL: https://islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}/cantaloupe/iiif/2 - DRUPAL_DEFAULT_FCREPO_HOST: fcrepo-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - DRUPAL_DEFAULT_DB_HOST: ${DRUPAL_DATABASE_SERVICE}-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - DRUPAL_DEFAULT_MATOMO_URL: https://islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me}/matomo/ - DRUPAL_DEFAULT_SITE_URL: http://islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} # Make sure this is just http and not https! - # - # Salt for one-time login links, cancel links, form tokens, etc, if deploying multiple - # instances of the site this must be set the same for all instances. - # - DRUPAL_DEFAULT_SALT: 9PPaL0CxZAIcq0l9wxgDGlCZrp7JdT_x7v9gVzpdbUjMt1PqDz3uD0Zy-i16DuJ1-Htuq5hqeg - # - # The administrator email account for the default site. - # - DRUPAL_DEFAULT_ACCOUNT_EMAIL: admin@example.org - # - # The password of the admin account for the default site. - # - DRUPAL_DEFAULT_ACCOUNT_PASSWORD: password - # - # The password Drupal uses to connect to the database for the default site. - # - DRUPAL_DEFAULT_DB_PASSWORD: password - # - # Must match MYSQL_ROOT_PASSWORD, used to create the database / user. - # - DRUPAL_DEFAULT_DB_ROOT_PASSWORD: password - # - # JWT Key Pair - # - DRUPAL_JWT_PRIVATE_KEY: |- - -----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA6uK3nozywVaRCAB3FHdRZNHunSZvN/c31QimZAqQMGxj7JrG - h1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2wARr/9abIwG5QeNsrJ8GMt3Z/WICn - NeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRKsk0BfFSMh+4u3JAdD9tUxUx6AAUX - UCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80Xrrsu1PYATgrsy69stzCln3KlO5O - xc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9Pa3SNje4UXN+/1xUfQskxafP8CKVS - r8xxtwzSureiskb5/98moAiutpUtp15yyAm0rwIDAQABAoIBAA0PZh5OwAC4C4Bi - ZjyhFcmBUr8yL+Twvg3+WSIe5D2NCVFSmc9UbuUdmnaoIIlrf61p6Vo88VCMVfWR - Z3iFj0/AbJMAHxF0EM1nglLHlEdvM018ec+pbaPeq4LTeA/dfGgDmcyQ53b1lO30 - KMt5st2PIpIDMX0tZTWmXbdP/rqplqiQmdwH0gv8PzEG6Y2ZVLBf5viH5IvVRpg7 - 9nDqLfe2W2ylFib+CtleX626xTUzGcJ0aqTRP1UkY4Jj5PI2/yVqttYPkvJtxZco - 5/14AEMcu9FMBhADCSk/0y1TkKCGsi6/VNd78AB/RrZK32HfHCwTwxoXHQoaVKq6 - hNQfokECgYEA+keGdVJrXMDylamARQdDe/nNgljqZZhfkKKYCGeckNqjp3iqfmld - /tqCPVxAO2mIo8dNNfM2MMv6loEPx9F2REe8k9NbWFrUZ486fMHPeO7WnHt92JyU - DtfJJSZ1GdCki1PthmpmP8WdoF6VpLvr5AgwuYKAzkMNth+OV/dvgqkCgYEA8EEe - E3mKvePHV/PVsLt6TaqJcZEKKu6L9EgeDyzv3zz4+2zG8MVctyUFyfSl4EIC/oJy - X0T5Tj1l4A3mPwZOJfQOkXnr9TaPNff1zjNx12RhUZjFtJU5V+Wn1ldtzs+XwCFc - x5O/B8LWYgV4bOixNlc6tTRq/m8Txvtde9vPa5cCgYEA3OOdnxRD31QHheFYbRPx - Eo0xPNae4VWvGmb2SYywmQPuplMQHot+Qvy1L9SoeAc3alzvHytta3nLy2NS+yc5 - +x9ZJxrGJt/bUR8PHqarJu+ch/VR54ih/8uhImGjvknvv2wuWZC0d5pA+RYheofE - tLgp0MCGUATMKC4HokmmqCkCgYBEjoBTlFIn33CJw3WNyeGbefdgZb/eAlYDbfTN - 5cfJDvAJZr/aAqdzR2hAecQ/mvaZw4V5dAgj8Fc6uRyjjVwNbngdwQm43km9X7VP - ktSAXw96Jjr8TbygPVNIUYhvBEPMOnjsJlfTkiB0thToFvpChF+nR37kfbPKCv5h - Epc8nwKBgQCdKyLi54Fm24nqEuZYbAxxGI9TVT7wJjoKGn64JWrXtX7xRltmJC3t - nLwNCojcbyG4kVB+Myzr2OEtFkO45j83GjrZ4O+jCuSj+AmCxEcc7xNA9cgu9usG - sQXdGmIIB0Cbk54OyHNdsZgZCXi9GTRF9uvYZKL9qktS+UZMJ1Xz/g== - -----END RSA PRIVATE KEY----- - # - # All other *_JWT_PUBLIC keys should match this value exactly. - # - DRUPAL_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - fcrepo: - environment: - # - # The password Fedora uses to connect to the database. - # - FCREPO_DB_PASSWORD: password - # - # Must match MYSQL_ROOT_PASSWORD, used to create the database / user. - # - FCREPO_DB_ROOT_PASSWORD: password - # - # Definitive name of token for as defined in syn-settings.xml. - # - FCREPO_JWT_ADMIN_TOKEN: islandora - # - # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # - FCREPO_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - gemini: - environment: - # - # The password Gemini uses to connect to the database. - # - GEMINI_DB_PASSWORD: password - # - # Must match MYSQL_ROOT_PASSWORD, used to create the database / user. - # - GEMINI_DB_ROOT_PASSWORD: password - # - # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. - # - GEMINI_JWT_ADMIN_TOKEN: islandora - # - # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # - GEMINI_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - homarus: - environment: - # - # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. - # - HOMARUS_JWT_ADMIN_TOKEN: islandora - # - # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # - HOMARUS_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - houdini: - environment: - # - # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. - # - HOUDINI_JWT_ADMIN_TOKEN: islandora - # - # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # - HOUDINI_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - hypercube: - environment: - # - # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. - # - HYPERCUBE_JWT_ADMIN_TOKEN: islandora - # - # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # - HYPERCUBE_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - matomo: - environment: - # - # The password Matomo uses to connect to the database. - # - MATOMO_DB_PASSWORD: password - # - # Must match MYSQL_ROOT_PASSWORD, used to create the database / user. - # - MATOMO_DB_ROOT_PASSWORD: password - # - # Administrator email. - # - MATOMO_USER_EMAIL: admin@example.org - # To regenerate a the `MATOMO_USER_PASS` you must use the following - # snippet of [PHP](https://matomo.org/faq/how-to/faq_191/). - # - # php -r 'echo password_hash(md5("password"), PASSWORD_DEFAULT) . "\n";' - # - # N.B. You must double up '$' characters to prevent them from being - # interpolated by docker-compose.yml - MATOMO_USER_PASS: $$2y$$10$$S38e7HPM9LI3aOIvcnRsfuMCm4ipNP572QsvbCK60upoHVJ61hMrS - # - # Must be exactly the same as DRUPAL_DEFAULT_SITE_URL without the protocol. - # - # If you are using an IP address with traefik.me, change the dot between COMPOSE_PROJECT_NAME and - # DRUPAL_SITE_HOST to a hyphen (i.e. ${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - # becomes ${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} - # - MATOMO_SITE_HOST: islandora-${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} - # - # Timezone of site. - # - MATOMO_SITE_TIMEZONE: America/Halifax - milliner: - # No environment variables require overriding. - recast: - environment: - # - # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. - # - RECAST_JWT_ADMIN_TOKEN: islandora - # - # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # - RECAST_JWT_PUBLIC_KEY: |- - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR - ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w - ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK - sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 - Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P - a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 - rwIDAQAB - -----END PUBLIC KEY----- - solr: - # No environment variables require overriding. diff --git a/docker-compose.traefik.yml b/docker-compose.traefik.yml deleted file mode 100644 index d15477ca7..000000000 --- a/docker-compose.traefik.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Allows traefik to work as the edge router for multiple projects. -# -# By default the rules will redirect to ${SERVICE}.${COMPOSE_PROJECT_NAME}.traefik.me -# if working on localhost and ${SERVICE}-${COMPOSE_PROJECT_NAME}-XX-XX-XX-XX.traefik.me -# if working remotely, where XX-XX-XX-XX is the dashed version of your IP address. -# -# For a traefik to be able to route traffic to a given container, that -# container needs to be on the `gateway` network, otherwise traefik will -# discover it via Docker but will not be able to redirect traffic to it. -version: "3.7" -networks: - gateway: - name: gateway - driver: bridge - internal: false -services: - traefik: - image: traefik:2.2.1 - container_name: traefik - # Do not set `api.insecure`, `api.dashboard`, `api.debug` to `true` in production. - # Also do not expose database 3306/5432, as an entry point. - command: > - --api.insecure=true - --api.dashboard=true - --api.debug=true - --entryPoints.http.address=:80 - --entryPoints.https.address=:443 - --entryPoints.mysql.address=:3306 - --entryPoints.postgresql.address=:5432 - --providers.docker - --providers.docker.network=gateway - --providers.docker.exposedByDefault=false - --providers.file.filename=/etc/traefik/tls.yml - '--providers.docker.defaultRule=Host(`{{ index .Labels "com.docker.compose.service" }}-{{ index .Labels "com.docker.compose.project" }}.${DRUPAL_SITE_HOST:-traefik.me}`,`{{ index .Labels "com.docker.compose.service" }}-{{ index .Labels "com.docker.compose.project" }}-${DRUPAL_SITE_HOST:-traefik.me}`)' - ports: - # Has to be bind mounted to 80/443 for assumptions - # around internal network alias to work as intended. - - 80:80 - - 443:443 - # Allows direct connection to the database for using - # drush on the host machine. Do not do in production. - - 3306:3306 - - 5432:5432 - # The Web UI (enabled by --api.insecure=true) - - ${TRAEFIK_WEB_UI_PORT-8080}:8080 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./tls.yml:/etc/traefik/tls.yml - - ./certs:/etc/ssl/traefik - labels: - # Do not expose in production. - - traefik.http.routers.api.service=api@internal - networks: - - gateway diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 000000000..2cccd7e35 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,141 @@ +# Troubleshooting + +This document lists common ISLE problems and their solutions. It is currently a stub and should be added to. + +## Logging + +To view the Drupal logs from the shell, from your `isle-dc` directory, enter `docker-compose logs -f drupal`. That last argument is the service name, which can be swapped out or can actually be multiple (e.g. `drupal solr fcrepo`) or it can be empty, in which case you will see ALL logs. + +## "Your clock is ahead/behind" certificate error + +Issue: Your browser will not let you access your site because it claims your computer's clock is incorrect, +even when your clock is correct. + +Solution: In your isle directory, run the following commands: + +``` +curl http://traefik.me/fullchain.pem -o certs/cert.pem +curl http://traefik.me/privkey.pem -o certs/privkey.pem +``` + +Then restart the containers. + +## Errors doing `drush config:import -y`: + +**Error:** + +```html +Site UUID in source storage does not match the target Storage. +``` + +This occurs when your configuration has come from a site other than the one that +is currently installed. You can run the following command to override the site +`uuid` so that you can import your configuration: + +```bash +make set-site-uuid +``` + +**Error:** + +```html +Entities exist of type Shortcut link and Shortcut set +Default. These entities need to be deleted before importing +``` + +These are entities created by the `standard` installation profile, you can delete +them with the following command in the codebase folder: + +```bash +make delete-shortcut-entities +``` + +**Error:** + +```bash +Error: Call to a member function getConfigDependencyName() on null in ... Entity/EntityDisplayBase.php on line 325 #0 ... /codebase/web/core/lib/Drupal/Core/Config/Entity/ConfigEntityBase.php(318): Drupal\Core\Entity\EntityDisplayBase->calculateDependencies() +``` + +There is some bug in the dependencies of the various modules. Until those +dependencies issues are resolved just rerun the command until they go away. + +```bash +drush config:import -y +``` + +## Errors when doing an `drush sql:dump`: + +**Error:** + +This arises from our use of [MariaDB] for the database in Docker, not matching +the same client on your host system. Which probably uses the [MySQL] clients +`mysqldump` executable. You can specify the following command using drush to get +around it: + +```bash +drush sql:dump --extra-dump="--column-statistics=0" > /tmp/dump.sql +``` + +Or you can use the provided method in the [Makefile](./Makefile) which should be +more portable. + +```bash +make database-dump DEST=/tmp/dump.sql +``` + +## Drupal can't connect to a valid Solr container: + +**Error:** + +The server configured at `/admin/config/search/search-api` shows a failed connection to a properly configured Solr container. + +This can sometimes be caused by Docker containers not inheriting the DNS configurations from the host machine. + +To fix edit `/etc/resolv.conf` in both the Drupal and the Solr containers by adding a valid DNS entry i.e. `nameserver 223.5.5.5` + + +## Image or other derivatives are not produced, due to insufficient timeout limits + +**Symptoms:** + +Houdini converts images which is needed to produce image derivatives. Output from `docker-compose logs -f houdini` +such as this which is repeats (even when no further media have been uploaded) is an indication that the timeout +is exceeded, and alpaca is re-attempting: + +``` +houdini_1 | [2022-01-05 21:41:03] app.INFO: Convert request. [] [] +houdini_1 | [2022-01-05 21:41:03] app.DEBUG: X-Islandora-Args: {"args":"-thumbnail 100x100"} [] +houdini_1 | [2022-01-05 21:41:03] app.DEBUG: Content Types: [] [] +houdini_1 | [2022-01-05 21:41:03] app.DEBUG: Content Type Chosen: {"type":"image/jpeg"} [] +houdini_1 | [2022-01-05 21:41:03] app.INFO: Imagemagick Command: {"cmd":"convert - -thumbnail 100x100 jpeg:-"} [] +... +``` + +Output from `docker-compose logs -f alpaca` like this, which shows that it has hit a timeout and is giving up: + +``` +alpaca_1 | 2022-01-05 21:42:52,863 | ERROR | nnector-houdini] | DefaultErrorHandler +| 56 - org.apache.camel.camel-core - 2.20.4 | Failed delivery for (MessageId: +queue_islandora-connector-houdini_ID_94ca62ced546-38129-1641418608853-3_6_-1_1_5 on ExchangeId: +ID-bea81bcc2a4e-1641418615223-3-11). Exhausted after delivery attempt: 11 caught: +java.net.SocketTimeoutException: Read timed out. Processed by failure processor: +FatalFallbackErrorHandler[Channel[Log(ca.islandora.alpaca.connector.derivative.DerivativeConnector) +[Error connecting generating derivative with http://houdini:8000/convert: ${exception.message} +``` + +To fix this: + + * edit your .env file, an increase ALPACA_HOUDINI_TIMEOUT (and other similar timeouts if necessary). Note these values are in milliseconds. + * make docker-compose.yml (this is necessary to pick up the change and re-write docker-compose.yml) + * restart containers + +## Uncaught PHP Exception InvalidArgumentException or Flysystem driver is missing + +Error message likely to see. +> FastCGI sent in stderr: "PHP message: Uncaught PHP Exception InvalidArgumentException: "A valid cache entry key is required. Use getAll() to get all table data." at /var/www/drupal/web/core/modules/views/src/ViewsData.php line 140" while reading response header from upstream + +A log of the error is needed for the fix to review. A full description of the issue and how the error triggered is on https://github.com/Islandora-Devops/isle-dc/pull/213. + +To fix this: + +* Run `make fix_views` diff --git a/sample.Dockerfile b/sample.Dockerfile index eda0904c9..c5a536b00 100644 --- a/sample.Dockerfile +++ b/sample.Dockerfile @@ -1,15 +1,20 @@ # syntax=docker/dockerfile:experimental ARG REPOSITORY=islandora ARG TAG=latest -FROM ${REPOSITORY}/drupal:${TAG} +FROM ${REPOSITORY}/drupal:${TAG} as step1 + +COPY codebase /build + # Remove runtime configuration and data (files, settings.php, etc) these will # either be mounted as volumes or generated on startup from environment variables. -RUN --mount=type=bind,source=codebase,target=/build \ - cp -r /build/* /var/www/drupal && \ +RUN cp -r /build/* /var/www/drupal && \ rm -fr /var/www/drupal/web/sites/default/files/* && \ bash -lc "remove_standard_profile_references_from_config" && \ find /var/www/drupal/web/sites -name "settings.php" -exec rm {} \; && \ chown -R nginx:nginx /var/www/drupal -COPY rootfs / +FROM ${REPOSITORY}/drupal:${TAG} as application +COPY --from=step1 --chown=nginx:nginx /var/www/drupal /var/www/drupal + +COPY build/rootfs / diff --git a/sample.env b/sample.env index 6f2dec3df..be0f23474 100644 --- a/sample.env +++ b/sample.env @@ -10,9 +10,14 @@ # See documentation for more details. ENVIRONMENT=demo +# Enable this to generate a docker-compose file that uses secrets. +# If you're running staging, CI, or production, set to true. +USE_SECRETS=true + ############################################################################### # Environment variables specific to composer. ############################################################################### + COMPOSE_HTTP_TIMEOUT=480 # Also used for naming services in traefik as well as defining network alias and urls. @@ -20,12 +25,6 @@ COMPOSE_HTTP_TIMEOUT=480 # See https://docs.docker.com/compose/reference/envvars/ COMPOSE_PROJECT_NAME=isle-dc -# If you are running locally, leave this. traefik.me resolves to localhost. -# If you are deploying to a remote server and you own a domain for it, place it here. -# If you have an IP but no domain, change this to X-X-X-X.traefik.me, where X-X-X-X -# is your IP address but with hyphens instead of dots. -DRUPAL_SITE_HOST=traefik.me - # Allows building custom image with buildkit. COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 @@ -33,23 +32,40 @@ DOCKER_BUILDKIT=1 # Dockerfile to use when building the custom project. PROJECT_DRUPAL_DOCKERFILE=Dockerfile -# Includes `traefik` as a service, if false assume we are sharing a traefik +# Includes `traefik` as a service, if false assume we are sharing a traefik # from another project. INCLUDE_TRAEFIK_SERVICE=true +TRAEFIK_LOG_LEVEL=ERROR + +# Should we use ACME to generate a SSL Certificate +USE_ACME=false +# Specify email to tie SSL Certificate to with ACME provider +ACME_EMAIL=your-email@example.com +# KeyType used for generating certificate private key. Allow value 'EC256', 'EC384', 'RSA2048', 'RSA4096', 'RSA8192'. +ACME_KEY_TYPE=RSA4096 + +# ACME Defaults for Let's Encrypt Service +# ACME_SERVER=https://acme-v02.api.letsencrypt.org/directory +# Default duration for the certificate is 90 days or 2,160 hours for Let's Encrypt +# ACME_CERT_DURATION=2160 +# ACME_EAB_KID= +# ACME_EAB_HMAC= # Includes `watchtower` as a service. -INCLUDE_WATCHTOWER_SERVICE=true +INCLUDE_WATCHTOWER_SERVICE=false # Includes `etcd` as a service. INCLUDE_ETCD_SERVICE=false +# Includes `code-server` as a service. +INCLUDE_CODE_SERVER_SERVICE=false + # Choose which database backend to use: mariadb/postgresql # Services that only support MySQL (Matomo) do not have the option to change. # Also at this time not all Drupal modules work with PostgresSQL, it is provided # as option here so it can be tested and fixed at some later date. DRUPAL_DATABASE_SERVICE=mariadb FCREPO_DATABASE_SERVICE=mariadb -GEMINI_DATABASE_SERVICE=mariadb # Repository to use for pulling isle-buildkit images, change to `local` # To use images you have built locally with isle-buildkit, or use your @@ -59,4 +75,106 @@ REPOSITORY=islandora # The version of the isle-buildkit images, non isle-buildkit images have # their versions specified explicitly in their respective docker-compose files. -TAG=latest +TAG=1.0.10 + +############################################################################### +# Exposed Containers & Ports +############################################################################### + +# Expose Cantaloupe at ${DOMAIN}/cantaloupe +EXPOSE_CANTALOUPE=true + +# Expose Matomo at ${DOMAIN}/matomo +EXPOSE_MATOMO=true + +# Expose Drupal at ${DOMAIN} +EXPOSE_DRUPAL=true + +# Expose MySQL over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_MYSQL=false +MYSQL_PORT=3306 + +# Expose Postgres over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_POSTGRES=false +POSTGRES_PORT=5432 + +# Expose the Traefik dashboard over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_TRAEFIK_DASHBOARD=false +TRAEFIK_DASHBOARD_PORT=8080 + +# Expose Fedora over the given port - Necessary for Canteloupe +EXPOSE_FEDORA=true +FEDORA_PORT=8081 + +# Expose Blazegraph over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_BLAZEGRAPH=false +BLAZEGRAPH_PORT=8082 + +# Expose Activemq over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_ACTIVEMQ=false +ACTIVEMQ_PORT=8161 + +# Expose SOLR over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_SOLR=false +SOLR_PORT=8983 + +# Expose Code Server over the given port - DO NOT EXPOSE THIS IN PRODUCTION +EXPOSE_CODE_SERVER=false +CODE_SERVER_PORT=8443 + +############################################################################### +# Global Environment variables +############################################################################### + +DOMAIN=islandora.traefik.me +SITE=https://${DOMAIN} +DISABLE_SYN=false +FEDORA_6=true +RESTART_POLICY=unless-stopped + +# PHP variables +PHP_MEMORY_LIMIT=256M +PHP_POST_MAX_SIZE=128M +PHP_UPLOAD_MAX_FILESIZE=128M +PHP_MAX_EXECUTION_TIME=30 + +# If you're just demoing or are starting from scratch, use this. +INSTALL_EXISTING_CONFIG=false +DRUPAL_INSTALL_PROFILE=standard + +# If you're installing from an existing codebase, uncomment this +#INSTALL_EXISTING_CONFIG=true +#DRUPAL_INSTALL_PROFILE=minimal + +# If you're experiencing timeouts in Alpaca, consider bumping these +# values for each connector. They are in milliseconds (1000 = 1 second). +ALPACA_OCR_TIMEOUT=300000 +ALPACA_FITS_TIMEOUT=300000 +ALPACA_HOUDINI_TIMEOUT=300000 +ALPACA_HOMARUS_TIMEOUT=300000 + +# Set Memory Limit for each container. +ACTIVEMQ_MEMORY_LIMIT=2G +ALPACA_MEMORY_LIMIT=2G +BLAZEGRAPH_MEMORY_LIMIT=5G +CANTALOUPE_MEMORY_LIMIT=5G +CRAYFITS_MEMORY_LIMIT=1G +DRUPAL_MEMORY_LIMIT=5G +FCREPO_MEMORY_LIMIT=5G +FITS_MEMORY_LIMIT=5G +HOMARUS_MEMORY_LIMIT=1G +HOUDINI_MEMORY_LIMIT=1G +HYPERCUBE_MEMORY_LIMIT=512M +IDE_MEMORY_LIMIT=5G +MARIADB_MEMORY_LIMIT=1G +MATOMO_MEMORY_LIMIT=1G +MILLINER_MEMORY_LIMIT=1G +RECAST_MEMORY_LIMIT=1G +SOLR_MEMORY_LIMIT=8G +TRAEFIK_MEMORY_LIMIT=8G +WATCHTOWER_MEMORY_LIMIT=2G + +# Configuration to enable the custom delegate script for Cantaloupe +CANTALOUPE_DELEGATE_SCRIPT_ENABLED=false +CANTALOUPE_DELEGATE_SCRIPT_PATHNAME=/opt/tomcat/bin/delegates.rb +CANTALOUPE_HTTPSOURCE_LOOKUP_STRATEGY=BasicLookupStrategy diff --git a/scripts/default.services.yml b/scripts/default.services.yml new file mode 100644 index 000000000..5ab4ec957 --- /dev/null +++ b/scripts/default.services.yml @@ -0,0 +1,202 @@ +parameters: + session.storage.options: + # Default ini options for sessions. + # + # Some distributions of Linux (most notably Debian) ship their PHP + # installations with garbage collection (gc) disabled. Since Drupal depends + # on PHP's garbage collection for clearing sessions, ensure that garbage + # collection occurs by using the most common settings. + # @default 1 + gc_probability: 1 + # @default 100 + gc_divisor: 100 + # + # Set session lifetime (in seconds), i.e. the grace period for session + # data. Sessions are deleted by the session garbage collector after one + # session lifetime has elapsed since the user's last visit. When a session + # is deleted, authenticated users are logged out, and the contents of the + # user's session is discarded. + # @default 200000 + gc_maxlifetime: 200000 + # + # Set session cookie lifetime (in seconds), i.e. the time from the session + # is created to the cookie expires, i.e. when the browser is expected to + # discard the cookie. The value 0 means "until the browser is closed". + # @default 2000000 + cookie_lifetime: 2000000 + # + # Drupal automatically generates a unique session cookie name based on the + # full domain name used to access the site. This mechanism is sufficient + # for most use-cases, including multi-site deployments. However, if it is + # desired that a session can be reused across different subdomains, the + # cookie domain needs to be set to the shared base domain. Doing so assures + # that users remain logged in as they cross between various subdomains. + # To maximize compatibility and normalize the behavior across user agents, + # the cookie domain should start with a dot. + # + # @default none + # cookie_domain: '.example.com' + # + # Set the session ID string length. The length can be between 22 to 256. The + # PHP recommended value is 48. See + # https://www.php.net/manual/session.security.ini.php for more information. + # This value should be kept in sync with + # \Drupal\Core\Session\SessionConfiguration::__construct() + # @default 48 + sid_length: 48 + # + # Set the number of bits in encoded session ID character. The possible + # values are '4' (0-9, a-f), '5' (0-9, a-v), and '6' (0-9, a-z, A-Z, "-", + # ","). The PHP recommended value is 6. See + # https://www.php.net/manual/session.security.ini.php for more information. + # This value should be kept in sync with + # \Drupal\Core\Session\SessionConfiguration::__construct() + # @default 6 + sid_bits_per_character: 6 + twig.config: + # Twig debugging: + # + # When debugging is enabled: + # - The markup of each Twig template is surrounded by HTML comments that + # contain theming information, such as template file name suggestions. + # - Note that this debugging markup will cause automated tests that directly + # check rendered HTML to fail. When running automated tests, 'debug' + # should be set to FALSE. + # - The dump() function can be used in Twig templates to output information + # about template variables. + # - Twig templates are automatically recompiled whenever the source code + # changes (see auto_reload below). + # + # For more information about debugging Twig templates, see + # https://www.drupal.org/node/1906392. + # + # Enabling Twig debugging is not recommended in production environments. + # @default false + debug: false + # Twig auto-reload: + # + # Automatically recompile Twig templates whenever the source code changes. + # If you don't provide a value for auto_reload, it will be determined + # based on the value of debug. + # + # Enabling auto-reload is not recommended in production environments. + # @default null + auto_reload: null + # Twig cache: + # + # By default, Twig templates will be compiled and stored in the filesystem + # to increase performance. Disabling the Twig cache will recompile the + # templates from source each time they are used. In most cases the + # auto_reload setting above should be enabled rather than disabling the + # Twig cache. + # + # Disabling the Twig cache is not recommended in production environments. + # @default true + cache: true + # File extensions: + # + # List of file extensions the Twig system is allowed to load via the + # twig.loader.filesystem service. Files with other extensions will not be + # loaded unless they are added here. For example, to allow a file named + # 'example.partial' to be loaded, add 'partial' to this list. To load files + # with no extension, add an empty string '' to the list. + # + # @default ['css', 'html', 'js', 'svg', 'twig'] + allowed_file_extensions: + - css + - html + - js + - svg + - twig + renderer.config: + # Renderer required cache contexts: + # + # The Renderer will automatically associate these cache contexts with every + # render array, hence varying every render array by these cache contexts. + # + # @default ['languages:language_interface', 'theme', 'user.permissions'] + required_cache_contexts: ['languages:language_interface', 'theme', 'user.permissions'] + # Renderer automatic placeholdering conditions: + # + # Drupal allows portions of the page to be automatically deferred when + # rendering to improve cache performance. That is especially helpful for + # cache contexts that vary widely, such as the active user. On some sites + # those may be different, however, such as sites with only a handful of + # users. If you know what the high-cardinality cache contexts are for your + # site, specify those here. If you're not sure, the defaults are fairly safe + # in general. + # + # For more information about rendering optimizations see + # https://www.drupal.org/developing/api/8/render/arrays/cacheability#optimizing + auto_placeholder_conditions: + # Max-age at or below which caching is not considered worthwhile. + # + # Disable by setting to -1. + # + # @default 0 + max-age: 0 + # Cache contexts with a high cardinality. + # + # Disable by setting to []. + # + # @default ['session', 'user'] + contexts: ['session', 'user'] + # Tags with a high invalidation frequency. + # + # Disable by setting to []. + # + # @default [] + tags: [] + # Cacheability debugging: + # + # Responses with cacheability metadata (CacheableResponseInterface instances) + # get X-Drupal-Cache-Tags, X-Drupal-Cache-Contexts and X-Drupal-Cache-Max-Age + # headers. + # + # For more information about debugging cacheable responses, see + # https://www.drupal.org/developing/api/8/response/cacheable-response-interface + # + # Enabling cacheability debugging is not recommended in production + # environments. + # @default false + http.response.debug_cacheability_headers: false + factory.keyvalue: {} + # Default key/value storage service to use. + # @default keyvalue.database + # default: keyvalue.database + # Collection-specific overrides. + # state: keyvalue.database + factory.keyvalue.expirable: {} + # Default key/value expirable storage service to use. + # @default keyvalue.database.expirable + # default: keyvalue.database.expirable + # Allowed protocols for URL generation. + filter_protocols: + - http + - https + - ftp + - news + - nntp + - tel + - telnet + - mailto + - irc + - ssh + - sftp + - webcal + - rtsp + + cors.config: + enabled: true + # Specify allowed headers, like 'x-allowed-header'. + allowedHeaders: [] + # Specify allowed request methods, specify ['*'] to allow all possible ones. + allowedMethods: [] + # Configure requests allowed from specific origins. + allowedOrigins: ['*'] + # Sets the Access-Control-Expose-Headers header. + exposedHeaders: false + # Sets the Access-Control-Max-Age header. + maxAge: false + # Sets the Access-Control-Allow-Credentials header. + supportsCredentials: false diff --git a/scripts/host_prep.sh b/scripts/host_prep.sh new file mode 100644 index 000000000..5d3788aa5 --- /dev/null +++ b/scripts/host_prep.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +ROOT_DIR='/var/idc-isle-dc/' + +cd ${ROOT_DIR} || exit 1 + +if [ -f /etc/server-type.conf ]; then + servertypename=$(cat /etc/server-type.conf) + if [ $servertypename == "test" ]; then + if [ ! -f /etc/domain.conf ]; then + echo "File Missing /etc/domain.conf" + exit 1 + fi + domainname=$(cat /etc/domain.conf) + + echo "" + echo "Server Type: $servertypename Domain Name: $domainname" + sed -i "s/DOMAIN=.*/DOMAIN=$domainname/g" .env + sed -i "s/INCLUDE_TRAEFIK_SERVICE=.*/INCLUDE_TRAEFIK_SERVICE=no/g" .env + sed -i "s#host:.*#host: 'https://${domainname}/'#g" islandora_workbench/islandora_workbench_demo_content/jhu_root_collections.yml + + if [ ! -f ${ROOT_DIR}certs/privkey.pem.bak ]; then + echo "Copying ${ROOT_DIR}certs/privkey.pem to ${ROOT_DIR}certs/privkey.pem.bak" + cp ${ROOT_DIR}certs/privkey.pem ${ROOT_DIR}certs/privkey.pem.bak || exit 1 + fi + sudo openssl rsa -in /etc/pki/tls/private/star_mse.jhu.edu.key -text > ${ROOT_DIR}certs/privkey.pem || exit 1 + + if [ ! -f ${ROOT_DIR}certs/cert.pem.bak ]; then + echo "Copying ${ROOT_DIR}certs/cert.pem to ${ROOT_DIR}certs/cert.pem.bak" + cp ${ROOT_DIR}certs/cert.pem ${ROOT_DIR}certs/cert.pem.bak || exit 1 + fi + sudo cp /etc/pki/tls/certs/star_mse.jhu.edu.pem ${ROOT_DIR}certs/cert.pem || exit 1 + + make -B docker-compose.yml + docker-composer down ; docker-compose up -d + sleep 10 + fi + # Check if the URL is up and has a valid SSL certificate + url="${domainname}" + if true | openssl s_client -connect ${url}:443 2>/dev/null | openssl x509 -noout -checkend 0; then + echo "Looks like the URL is up and has a valid SSL certificate" + else + echo "Warning: URL is not available or has an invalid SSL certificate" + echo "Rolling back to the backup certs" + if [ -f ${ROOT_DIR}certs/privkey.pem.bak ]; then + mv ${ROOT_DIR}certs/privkey.pem.bak ${ROOT_DIR}certs/privkey.pem + fi + if [ -f ${ROOT_DIR}certs/cert.pem.bak ]; then + mv ${ROOT_DIR}certs/cert.pem.bak ${ROOT_DIR}certs/cert.pem + fi + + docker-compose restart traefik || exit 1 + fi +fi diff --git a/scripts/patch_views.sh b/scripts/patch_views.sh new file mode 100644 index 000000000..5c8bcbe2c --- /dev/null +++ b/scripts/patch_views.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -e + +# To be precise on the error message that matches the error this should address. +ERROR_MESSAGE=$(drush watchdog:show --severity=Error --filter="InvalidArgumentException: A valid cache entry key is required" | awk '{print $6}') + +# If error message equals to "No such file or directory", then exit. +if [[ $ERROR_MESSAGE == *'InvalidArgumentException'* ]]; then + + # Install Drupal Console. + drupal_console_installed() { + composer show 'drupal/console' | grep -q '/var/www/drupal/vendor/drupal/console' + } + if drupal_console_installed; then + echo 'Package installed' + else + composer require drupal/console:~1.0 --prefer-dist --optimize-autoloader -W + fi + + # Reinstall views + enabled_view=`/var/www/drupal/vendor/drupal/console/bin/drupal debug:views --status='Enabled' | cut -d ' ' -f 2 | tail -n +2` + for dis_view in $enabled_view; do + echo "Disabling view $dis_view" + /var/www/drupal/vendor/drupal/console/bin/drupal views:disable $dis_view + done + + for en_view in $enabled_view; do + echo "Reenabling view $en_view" + /var/www/drupal/vendor/drupal/console/bin/drupal views:enable $en_view + done + + # Install devel. + devel_installed() { + composer show 'drupal/devel' | grep -q '/var/www/drupal/web/modules/contrib/devel' + } + if devel_installed; then + echo 'Package installed' + else + composer require 'drupal/devel:^4.1' -W + fi + drush pm:enable -y devel + + echo -e "\n\nThis will likely throw an error, but that's okay. It's just a patch.\n\n" + { # try + drush dev:reinstall -y islandora + } || { # catch + echo -e "\nIgnore these errors. This will fail if any content is already created.\n\n" + } + + # Clear caches + /var/www/drupal/vendor/drupal/console/bin/drupal cache:rebuild + /var/www/drupal/vendor/drupal/console/bin/drupal cr all + /var/www/drupal/vendor/drupal/console/bin/drupal node:access:rebuild + drush cron +fi diff --git a/scripts/services.yml b/scripts/services.yml new file mode 100644 index 000000000..56da8aba2 --- /dev/null +++ b/scripts/services.yml @@ -0,0 +1,4 @@ +parameters: + twig.config: + debug: true + auto_reload: true \ No newline at end of file diff --git a/secrets/live/.keep b/secrets/live/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/secrets/template/ACTIVEMQ_PASSWORD b/secrets/template/ACTIVEMQ_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/ACTIVEMQ_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/ACTIVEMQ_WEB_ADMIN_PASSWORD b/secrets/template/ACTIVEMQ_WEB_ADMIN_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/ACTIVEMQ_WEB_ADMIN_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/ALPACA_ACTIVEMQ_PASSWORD b/secrets/template/ALPACA_ACTIVEMQ_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/ALPACA_ACTIVEMQ_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/ALPACA_KARAF_ADMIN_PASSWORD b/secrets/template/ALPACA_KARAF_ADMIN_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/ALPACA_KARAF_ADMIN_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/CODE_SERVER_PASSWORD b/secrets/template/CODE_SERVER_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/CODE_SERVER_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/DB_ROOT_PASSWORD b/secrets/template/DB_ROOT_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/DB_ROOT_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/DRUPAL_DEFAULT_ACCOUNT_PASSWORD b/secrets/template/DRUPAL_DEFAULT_ACCOUNT_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/DRUPAL_DEFAULT_ACCOUNT_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/DRUPAL_DEFAULT_CONFIGDIR b/secrets/template/DRUPAL_DEFAULT_CONFIGDIR new file mode 100644 index 000000000..582f04f76 --- /dev/null +++ b/secrets/template/DRUPAL_DEFAULT_CONFIGDIR @@ -0,0 +1 @@ +/var/www/drupal/config/sync \ No newline at end of file diff --git a/secrets/template/DRUPAL_DEFAULT_DB_PASSWORD b/secrets/template/DRUPAL_DEFAULT_DB_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/DRUPAL_DEFAULT_DB_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/DRUPAL_DEFAULT_SALT b/secrets/template/DRUPAL_DEFAULT_SALT new file mode 100644 index 000000000..85d2d75c1 --- /dev/null +++ b/secrets/template/DRUPAL_DEFAULT_SALT @@ -0,0 +1 @@ +9PPaL0CxZAIcq0l9wxgDGlCZrp7JdT_x7v9gVzpdbUjMt1PqDz3uD0Zy-i16DuJ1-Htuq5hqeg \ No newline at end of file diff --git a/secrets/template/FCREPO_DB_PASSWORD b/secrets/template/FCREPO_DB_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/FCREPO_DB_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/GEMINI_DB_PASSWORD b/secrets/template/GEMINI_DB_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/GEMINI_DB_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/JWT_ADMIN_TOKEN b/secrets/template/JWT_ADMIN_TOKEN new file mode 100644 index 000000000..550637dc5 --- /dev/null +++ b/secrets/template/JWT_ADMIN_TOKEN @@ -0,0 +1 @@ +islandora \ No newline at end of file diff --git a/secrets/template/JWT_PRIVATE_KEY b/secrets/template/JWT_PRIVATE_KEY new file mode 100644 index 000000000..37d25c7a1 --- /dev/null +++ b/secrets/template/JWT_PRIVATE_KEY @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA6uK3nozywVaRCAB3FHdRZNHunSZvN/c31QimZAqQMGxj7JrG +h1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2wARr/9abIwG5QeNsrJ8GMt3Z/WICn +NeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRKsk0BfFSMh+4u3JAdD9tUxUx6AAUX +UCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80Xrrsu1PYATgrsy69stzCln3KlO5O +xc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9Pa3SNje4UXN+/1xUfQskxafP8CKVS +r8xxtwzSureiskb5/98moAiutpUtp15yyAm0rwIDAQABAoIBAA0PZh5OwAC4C4Bi +ZjyhFcmBUr8yL+Twvg3+WSIe5D2NCVFSmc9UbuUdmnaoIIlrf61p6Vo88VCMVfWR +Z3iFj0/AbJMAHxF0EM1nglLHlEdvM018ec+pbaPeq4LTeA/dfGgDmcyQ53b1lO30 +KMt5st2PIpIDMX0tZTWmXbdP/rqplqiQmdwH0gv8PzEG6Y2ZVLBf5viH5IvVRpg7 +9nDqLfe2W2ylFib+CtleX626xTUzGcJ0aqTRP1UkY4Jj5PI2/yVqttYPkvJtxZco +5/14AEMcu9FMBhADCSk/0y1TkKCGsi6/VNd78AB/RrZK32HfHCwTwxoXHQoaVKq6 +hNQfokECgYEA+keGdVJrXMDylamARQdDe/nNgljqZZhfkKKYCGeckNqjp3iqfmld +/tqCPVxAO2mIo8dNNfM2MMv6loEPx9F2REe8k9NbWFrUZ486fMHPeO7WnHt92JyU +DtfJJSZ1GdCki1PthmpmP8WdoF6VpLvr5AgwuYKAzkMNth+OV/dvgqkCgYEA8EEe +E3mKvePHV/PVsLt6TaqJcZEKKu6L9EgeDyzv3zz4+2zG8MVctyUFyfSl4EIC/oJy +X0T5Tj1l4A3mPwZOJfQOkXnr9TaPNff1zjNx12RhUZjFtJU5V+Wn1ldtzs+XwCFc +x5O/B8LWYgV4bOixNlc6tTRq/m8Txvtde9vPa5cCgYEA3OOdnxRD31QHheFYbRPx +Eo0xPNae4VWvGmb2SYywmQPuplMQHot+Qvy1L9SoeAc3alzvHytta3nLy2NS+yc5 ++x9ZJxrGJt/bUR8PHqarJu+ch/VR54ih/8uhImGjvknvv2wuWZC0d5pA+RYheofE +tLgp0MCGUATMKC4HokmmqCkCgYBEjoBTlFIn33CJw3WNyeGbefdgZb/eAlYDbfTN +5cfJDvAJZr/aAqdzR2hAecQ/mvaZw4V5dAgj8Fc6uRyjjVwNbngdwQm43km9X7VP +ktSAXw96Jjr8TbygPVNIUYhvBEPMOnjsJlfTkiB0thToFvpChF+nR37kfbPKCv5h +Epc8nwKBgQCdKyLi54Fm24nqEuZYbAxxGI9TVT7wJjoKGn64JWrXtX7xRltmJC3t +nLwNCojcbyG4kVB+Myzr2OEtFkO45j83GjrZ4O+jCuSj+AmCxEcc7xNA9cgu9usG +sQXdGmIIB0Cbk54OyHNdsZgZCXi9GTRF9uvYZKL9qktS+UZMJ1Xz/g== +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/secrets/template/JWT_PUBLIC_KEY b/secrets/template/JWT_PUBLIC_KEY new file mode 100644 index 000000000..fcf182040 --- /dev/null +++ b/secrets/template/JWT_PUBLIC_KEY @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR +ZNHunSZvN/c31QimZAqQMGxj7JrGh1LF8JRX+XAQ+CJcPD9r6xXjKSS1Gqa2Os2w +ARr/9abIwG5QeNsrJ8GMt3Z/WICnNeaFAkUVviwKWcA61iFJWvTDAuI0hCaxArRK +sk0BfFSMh+4u3JAdD9tUxUx6AAUXUCdtPyluaBd53wuB0r9xRlPnDw6I9QHfKK80 +Xrrsu1PYATgrsy69stzCln3KlO5Oxc6O8OjMdjC2D2c3HmsO4CKPvvaVuaow/a9P +a3SNje4UXN+/1xUfQskxafP8CKVSr8xxtwzSureiskb5/98moAiutpUtp15yyAm0 +rwIDAQAB +-----END PUBLIC KEY----- \ No newline at end of file diff --git a/secrets/template/MATOMO_DB_PASSWORD b/secrets/template/MATOMO_DB_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/MATOMO_DB_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/secrets/template/MATOMO_USER_PASS b/secrets/template/MATOMO_USER_PASS new file mode 100644 index 000000000..71bfb6d67 --- /dev/null +++ b/secrets/template/MATOMO_USER_PASS @@ -0,0 +1 @@ +$2y$10$QnpdLINFB5hTuiz2XfJSN.3Wxe4UDlhblagmUXIiIEFPiMDVMYr7O \ No newline at end of file diff --git a/secrets/template/TOMCAT_ADMIN_PASSWORD b/secrets/template/TOMCAT_ADMIN_PASSWORD new file mode 100644 index 000000000..7aa311adf --- /dev/null +++ b/secrets/template/TOMCAT_ADMIN_PASSWORD @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/temp_migrate_fix.sh b/temp_migrate_fix.sh new file mode 100755 index 000000000..d050205dd --- /dev/null +++ b/temp_migrate_fix.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# For each title find the Node ID and replace the "collection_object:$TITLE" with the Node ID in the CSV file. +# NODE_ID=$(docker-compose exec -T drupal with-contenv bash -lc 'drush ev '\''$nodes = \Drupal::entityTypeManager()->getStorage("node")->loadByProperties(["title" => "$TITLE[0]"]); if (!empty($nodes)) { print reset($nodes)->id(); } else { print "No node found with the given title."; }'\''') +# sed -i 's/"collection_object:Johns Hopkins File 7 (Television program)"/$NODE_ID/g' islandora_workbench/islandora_workbench_demo_content/migration_from_production/islandora_objects.yml +echo -e "\n\n\nRunning Script to replace the collection_object:$TITLE with the Node ID in the CSV file" + +# Define the titles in an array +TITLES=( + "Johns Hopkins File 7 (Television program)" + "Johns Hopkins science review (Television program)" + "Johns Hopkins University graphic and pictorial collection" + "Johns Hopkins University maps and atlases" + "Johns Hopkins University News-Letter" + "Johns Hopkins University oral history collection" + "Johns Hopkins University television programs collection" + "Tomorrow (Television program)" + "Tomorrow's careers (Television program)" +) + +# Define the file path +FILE_PATH="islandora_workbench/islandora_workbench_demo_content/migration_from_production/islandora_objects.csv" + +# Loop through each title +for TITLE in "${TITLES[@]}"; do + echo "Fetch the Node ID for the current title in the CSV file" + ESCAPED_TITLE=$(echo "$TITLE" | sed "s/'/'\\\\''/g") # Escape single quotes in the title + NODE_ID=$(docker-compose exec -T drupal with-contenv bash -lc "drush ev '\$nodes = \Drupal::entityTypeManager()->getStorage(\"node\")->loadByProperties([\"title\" => \"$ESCAPED_TITLE\"]); if (!empty(\$nodes)) { print reset(\$nodes)->id(); } else { print \"No node found with the given title.\"; }'") + + # Check if NODE_ID is numeric (meaning we found a valid node ID) + if [[ $NODE_ID =~ ^[0-9]+$ ]]; then + echo "Replace the collection_object:$TITLE with the Node ID: $NODE_ID in the file" + sed -i "s/,\"collection_object:$TITLE\",/,$NODE_ID,/g" "$FILE_PATH" + sed -i "s/,collection_object:$TITLE,/,$NODE_ID,/g" "$FILE_PATH" + else + echo "No Node ID found for title: $TITLE" + fi +done + +echo -e "\ndone.\n\n\n"