Update for Document Engine 1.4.0 (#14)

.github/workflows/lint-test.yaml

@@ -44,4 +44,5 @@ jobs:
         if: steps.list-changed.outputs.changed == 'true'
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
-          ct install --target-branch ${{ github.event.repository.default_branch }} --helm-extra-set-args "--set=pspdfkit.storage.postgres.enabled=false"
+          ct install --target-branch ${{ github.event.repository.default_branch }} \
+             --helm-extra-args "--timeout 600s"

charts/document-engine/CHANGELOG.md

@@ -1,27 +1,58 @@
 # Changelog
 
 - [Changelog](#changelog)
-  - [2.7.3](#273)
+  - [2.8.0](#280)
+    - [Added](#added)
     - [Changed](#changed)
-  - [2.7.2](#272)
     - [Fixed](#fixed)
-  - [2.7.0](#270)
+  - [2.7.3](#273)
     - [Changed](#changed-1)
-  - [2.6.2](#262)
-    - [Added](#added)
+    - [Fixed](#fixed-1)
+  - [2.7.2](#272)
+    - [Fixed](#fixed-2)
+  - [2.7.0](#270)
     - [Changed](#changed-2)
-  - [2.6.0](#260)
+  - [2.6.2](#262)
     - [Added](#added-1)
-  - [2.4.0](#240)
+    - [Changed](#changed-3)
+  - [2.6.0](#260)
     - [Added](#added-2)
-  - [2.3.0](#230)
+  - [2.4.0](#240)
     - [Added](#added-3)
-  - [2.2.0](#220)
+  - [2.3.0](#230)
     - [Added](#added-4)
+  - [2.2.0](#220)
+    - [Added](#added-5)
   - [2.1.0](#210)
-    - [Changed](#changed-3)
-  - [2.0.0](#200)
     - [Changed](#changed-4)
+  - [2.0.0](#200)
+    - [Changed](#changed-5)
+
+## 2.8.0
+
+### Added
+
+* [Document Engine 1.4.0](https://pspdfkit.com/changelog/document-engine/#1.4.0)
+* Support for OpenTelemetry traces, enabled by setting `pspdfkit.observability.opentelemetry.enabled` to `true`. 
+  * Unless the collector is placed as a sidecar and receives by grpc at port `4317`, other parameters are also necessary. 
+  * Please note: standard OpenTelemetry environment variables are used, and the following values are just convenience wrappers, therefore other configuration approaches (e.g. setting variables through mutations or post build patches) will also work.
+  * Wrapped parameters (see `values.yaml` for more details):
+    * `pspdfkit.observability.opentelemetry.otlpExporterEndpoint` (`OTEL_EXPORTER_OTLP_ENDPOINT`)
+    * `pspdfkit.observability.opentelemetry.otlpExporterProtocol` (`OTEL_EXPORTER_OTLP_PROTOCOL`)
+    * `pspdfkit.observability.opentelemetry.otelServiceName` (`OTEL_SERVICE_NAME`)
+    * `pspdfkit.observability.opentelemetry.otelResourceAttributes` (`OTEL_RESOURCE_ATTRIBUTES`)
+    * `pspdfkit.observability.opentelemetry.otelPropagators` (`OTEL_PROPAGATORS`)
+* Dependent charts for [MinIO](https://min.io/) and [Redis](https://redis.io/).
+
+### Changed
+
+* Changed `pspdfkit.storage.enableMigrationJobs` to `pspdfkit.storage.databaseMigrationJob.enabled`.
+* Renamed `.Values.pspdfkit.storage.redis.sentinels` to `.Values.pspdfkit.storage.redis.sentinel`.
+* Slight refinement of trust information parameters: all files from `pspdfkit.trustConfigMaps` are now mounted to `/certificate-stores-custom/` to avoid confusion with `/certificate-stores/` which services for document signature validation certificates.
+
+### Fixed
+
+* Minor cleanups.
 
 ## 2.7.3
 

charts/document-engine/Chart.yaml

@@ -3,8 +3,8 @@ name: document-engine
 description: PSPDFKit Document Engine
 icon: data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTAwMCIgaGVpZ2h0PSIxMDAwIiB2aWV3Qm94PSIwIDAgMTAwMCAxMDAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgo8cGF0aCBkPSJNMzQyLjU2NyA2NTguNjEyTDMyOS40OTYgNjY1LjkxNUwzMzguNzk5IDY1NC4yODZDMzk5LjMxIDU3OS44NDIgNDMzLjAyNiA0ODcuMjExIDQzNC41MjggMzkxLjI4OEw0MzQuNzYgMzc2LjMxTDQ0MC4xMSAzOTAuMjY1QzQ3NC40MDUgNDc5LjkzMiA1MzcuODc4IDU1NS40NDggNjIwLjMxIDYwNC42NTRMNjMzLjE0OCA2MTIuMzI5TDYxOC4zNTYgNjA5Ljk1N0M1MjMuNTcxIDU5NC43NTYgNDI2LjQyOCA2MTEuODk0IDM0Mi41NjcgNjU4LjYxMloiIGZpbGw9IiMxOTBEOTQiLz4KPHBhdGggZD0iTTU1NC41MTkgNTU2LjM2N0M0NzMuMzk2IDQ0Mi4wMzMgNDg3LjMwNCAzNzYuODY1IDUwOS44NjQgMjg5Ljc4OEM1MzYuNjExIDE4Ni41MjQgNDczLjM1IDg0LjA1MDUgMzYyLjU5NyAxMTQuNDI1QzMxMy41MjMgMTI3LjkxNCAyOTYuMTczIDE2MS4zMTIgMjkwLjAzMyAxODQuNzU2QzI1OS45ODQgMjk3Ljc4OSA0MzguMjMxIDQ2Ni4xNzQgNTU0LjUxOSA1NTYuMzY3WiIgZmlsbD0iIzVFNUNFQiIvPgo8cGF0aCBkPSJNMjE3LjEwMyA2NDQuMTg2QzExNC4zMDQgNjcyLjY1NCA1Ny4xODM1IDc3OC43NTUgMTM4Ljg2NSA4NTkuNDEzQzE3NS4wNTMgODk1LjEzNyAyMTIuNjM4IDg5My40NjIgMjM2LjAzNSA4ODcuMzIyQzM0OC45MjggODU2Ljc2MiA0MDUuNjMgNjE4LjE4NCA0MjUuNjMyIDQ3Mi40MDVDMzY3LjE2MiA1OTkuNjI1IDMwMy45OTQgNjIwLjE4NCAyMTcuMTAzIDY0NC4xODZaIiBmaWxsPSIjNUU1Q0VCIi8+CjxwYXRoIGQ9Ik04NzEuNDggNTgyLjY1MUM3ODguNTQzIDUwMC4xNzkgNTUzLjU5NCA1NzAuMzI0IDQxNy4zNSA2MjUuOTFDNTU2Ljg5NiA2MTIuODM5IDYwNi4zODkgNjU3LjQ0OCA2NzAuNTMzIDcyMC41MjJDNzQ2LjU4NiA3OTUuMzE5IDg2Ni45NjggNzkxLjczNyA4OTYuMDQgNjgwLjY1OUM5MDguNjkyIDYzMS40NDUgODg4LjQ1OCA1OTkuNzIyIDg3MS40OCA1ODIuNjUxWiIgZmlsbD0iIzVFNUNFQiIvPgo8L3N2Zz4K
 type: application
-version: 2.7.4
-appVersion: "1.3.0"
+version: 2.8.0
+appVersion: "1.4.0"
 
 maintainers:
   - name: PSPDFKit
@@ -13,6 +13,14 @@ maintainers:
 
 dependencies:
   - name: postgresql
-    version: 15.5.9
+    version: 15.5.20
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
+  - name: minio
+    version: 14.6.29
+    repository: https://charts.bitnami.com/bitnami
+    condition: minio.enabled
+  - name: redis
+    version: 19.6.4
+    repository: https://charts.bitnami.com/bitnami
+    condition: redis.enabled

charts/document-engine/README.md

@@ -3,7 +3,9 @@
 - [Document Engine Helm chart](#document-engine-helm-chart)
   - [Using this repository](#using-this-repository)
   - [Installing Document Engine](#installing-document-engine)
-  - [Upgrade](#upgrade)
+    - [Dependencies](#dependencies)
+    - [Upgrade](#upgrade)
+  - [Contribution](#contribution)
   - [License](#license)
   - [Support, Issues and License Questions](#support-issues-and-license-questions)
 
@@ -19,18 +21,30 @@ helm repo update
 
 ## Installing Document Engine
 
-```
-helm upgrade --install --debug --dry-run \
-     pspdfkit/document-engine \
+```shell
+helm upgrade --install \
+     document-engine pspdfkit/document-engine \
      -n pspdfkit-services \
      -f ./document-engine-values.yaml
 ```
 
-## Upgrade
+### Dependencies
+
+The chart depends upon [Bitnami](https://github.com/bitnami/charts/tree/main/bitnami) charts for PostgreSQL, [MinIO](https://min.io/) and [Redis](https://redis.io/). They are disabled by default, but can be enabled for convenience. Please consider [tests](/charts/document-engine/ci) as examples.
+
+### Upgrade
 
 > [!NOTE] 
 > Please consult the [changelog](/charts/document-engine/CHANGELOG.md)
 
+## Contribution
+
+The chart is validated using [ct](https://github.com/helm/chart-testing/tree/main) [lint](https://github.com/helm/chart-testing/blob/main/doc/ct_lint.md):
+
+```shell
+ct lint --target-branch "$(git rev-parse --abbrev-ref HEAD)"
+```
+
 ## License
 
 This software is licensed under a [modified BSD license](LICENSE).

charts/document-engine/ci/1-no-storage-values.yaml

@@ -0,0 +1,13 @@
+pspdfkit:
+  storage:
+    postgres:
+      enabled: false
+
+postgresql:
+  enabled: false
+
+minio:
+  enabled: false
+
+redis:
+  enabled: false

charts/document-engine/ci/2-with-db-values.yaml

@@ -0,0 +1,33 @@
+pspdfkit:
+  storage:
+    databaseMigrationJob:
+      enabled: false
+    assetStorageBackend: built-in
+    postgres:
+      enabled: true
+      host: postgresql
+      port: 5432
+      username: postgres
+
+prometheusExporter:
+  enabled: true
+
+postgresql:
+  enabled: true
+  fullnameOverride: postgresql
+  nameOverride: postgresql
+  primary:
+    persistence:
+      enabled: true
+      size: 1Gi
+      storageClass: standard
+      accessModes:
+        - ReadWriteOnce
+  volumePermissions:
+    enabled: true
+
+minio:
+  enabled: false
+
+redis:
+  enabled: false

charts/document-engine/ci/3-with-db-s3-redis-values.yaml

@@ -0,0 +1,98 @@
+pspdfkit:
+  storage:
+    databaseMigrationJob:
+      enabled: false
+      podAnnotations:
+        do: "this"
+      podLabels:
+        do: "that"
+    assetStorageBackend: s3
+    postgres:
+      enabled: true
+      host: postgresql
+      port: 5432
+      username: postgres
+    s3:
+      accessKeyId: "pspdfkitObjectStorageRootKey"
+      secretAccessKey: "pspdfkitObjectStorageRootPassword"
+      bucket: "document-engine-assets"
+      region: "us-east-1"
+      host: "minio"
+      port: 9000
+      scheme: "http://"
+    redis:
+      enabled: true
+      host: redis-master
+      database: ""
+      username: ""
+      password: ""
+      ttlSeconds: 3600
+      useTtlForPrerendering: true
+      sentinel:
+        enabled: false
+        urls:
+          - redis://redis-node-0:26379
+          - redis://redis-node-1:26379
+          - redis://redis-node-2:26379
+        group: pspdfkit
+
+prometheusExporter:
+  enabled: true
+
+postgresql:
+  enabled: true
+  fullnameOverride: postgresql
+  nameOverride: postgresql
+  primary:
+    persistence:
+      enabled: true
+      size: 1Gi
+      storageClass: standard
+      accessModes:
+        - ReadWriteOnce
+  volumePermissions:
+    enabled: true
+
+minio:
+  enabled: true
+  fullnameOverride: minio
+  nameOverride: minio
+  auth:
+    rootUser: pspdfkitObjectStorageRootKey
+    rootPassword: pspdfkitObjectStorageRootPassword
+  defaultBuckets: "document-engine-assets"
+
+redis:
+  enabled: true
+  nameOverride: redis
+  fullnameOverride: redis
+  architecture: standalone
+  auth:
+    enabled: false
+    sentinel: false
+    password: pspdfkit
+  master:
+    count: 1
+    revisionHistoryLimit: 1
+    persistence:
+      enabled: false
+    persistentVolumeClaimRetentionPolicy:
+      enabled: false
+      whenDeleted: Delete
+  # replica:
+  #   replicaCount: 2
+  #   revisionHistoryLimit: 1
+  #   persistence:
+  #     enabled: false
+  #   persistentVolumeClaimRetentionPolicy:
+  #     enabled: false
+  #     whenDeleted: Delete
+  # sentinel:
+  #   enabled: true
+  #   quorum: 3
+  #   masterSet: pspdfkit
+  #   getMasterTimeout: 90
+  #   automateClusterRecovery: true
+  #   redisShutdownWaitFailover: true
+  #   persistence:
+  #     enabled: false

charts/document-engine/ci/README.md

@@ -0,0 +1,5 @@
+# Tests
+
+* `1-no-storage-values.yaml` — Document Engine with no storage, processing only
+* `2-with-db-values.yaml` — Basic configuration with PostgreSQL database
+* `3-with-db-s3-redis-values.yaml` — PostgreSQL, MinIO as S3 asset storage backend and Redis for rendering cache

charts/document-engine/templates/_helpers.tpl

@@ -55,6 +55,11 @@ document-engine.pspdfkit/job: cleanup
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{- define "document-engine.migrationSelectorLabels" -}}
+document-engine.pspdfkit/job: migration
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}

charts/document-engine/templates/config.ConfigMap.yaml

@@ -22,7 +22,7 @@ data:
   AUTOMATIC_LINK_EXTRACTION: {{ .Values.pspdfkit.automaticLinkExtraction | quote }}
   MIN_SEARCH_QUERY_LENGTH: {{ .Values.pspdfkit.minSearchQueryLength | quote }}
   DATABASE_CONNECTIONS: {{ .Values.pspdfkit.storage.databaseConnections | quote }}
-{{- if .Values.pspdfkit.storage.enableMigrationJobs }}
+{{- if .Values.pspdfkit.storage.databaseMigrationJob.enabled }}
   ENABLE_DATABASE_MIGRATIONS: "false"
   EXIT_AFTER_DATABASE_MIGRATIONS: "false"
 {{- else }}
@@ -69,5 +69,27 @@ data:
   STATSD_PORT: {{ .Values.prometheusExporter.statsdPort | quote }}
   STATSD_CUSTOM_TAGS: "namespace={{ .Release.Namespace }},app={{ include "document-engine.fullname" . }}"
 {{- end }}
+{{- with .Values.pspdfkit.observability.opentelemetry }}
+{{-   if .enabled }}
+  ENABLE_OPENTELEMETRY: "true"
+{{-     with .otlpExporterEndpoint }}
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ . | quote }}
+{{-     end }}
+{{-     with .otlpExporterProtocol }}
+  OTEL_EXPORTER_OTLP_PROTOCOL: {{ . | quote }}
+{{-     end }}
+{{-     with .otelServiceName }}
+  OTEL_SERVICE_NAME: {{ . | quote }}
+{{-     end }}
+{{-     with .otelResourceAttributes }}
+  OTEL_RESOURCE_ATTRIBUTES: {{ . | quote }}
+{{-     end }}
+{{-     with .otelPropagators }}
+  OTEL_PROPAGATORS: {{ . | quote }}
+{{-     end }}
+{{-   else }}
+  ENABLE_OPENTELEMETRY: "false"
+{{-   end }}
+{{- end }}
 # Can be used for hash updating
-  VERSION: {{ include "document-engine.chart" . | quote }}
+  HELM_CHART_VERSION: {{ include "document-engine.chart" . | quote }}

charts/document-engine/templates/config.storage-redis.Secret.yaml

@@ -7,16 +7,22 @@ metadata:
     {{- include "document-engine.labels" . | nindent 4 }}
 data:
 {{- with .Values.pspdfkit.storage.redis }}
-{{-     if .sentinels.enabled }}
-  REDIS_SENTINELS: {{ .sentinels.urls | join ";" | b64enc | quote }}
-  REDIS_SENTINELS_GROUP: {{ .sentinels.group | b64enc | quote }}
-{{-     else }}
+{{-   if .sentinel.enabled }}
+  REDIS_SENTINELS: {{ .sentinel.urls | join ";" | b64enc | quote }}
+  REDIS_SENTINELS_GROUP: {{ .sentinel.group | b64enc | quote }}
+{{-   else }}
   REDIS_HOST: {{ .host | b64enc | quote }}
-  REDIS_PORT: {{ .port | toString | b64enc | quote }}
   REDIS_SSL: {{ .tls.enabled | toString | b64enc | quote }}
-{{-     end }}
-  REDIS_DATABASE: {{ .database | b64enc | quote }}
-  REDIS_USERNAME: {{ .username | b64enc | quote }}
-  REDIS_PASSWORD: {{ .password | b64enc | quote }}
+{{-   end }}
+  REDIS_PORT: {{ .port | toString | b64enc | quote }}
+{{-   with .database }}
+  REDIS_DATABASE: {{ . | b64enc | quote }}
+{{-   end }}
+{{-   with .username }}
+  REDIS_USERNAME: {{ . | b64enc | quote }}
+{{-   end }}
+{{-   with .password }}
+  REDIS_PASSWORD: {{ . | b64enc | quote }}
+{{-   end }}
 {{- end }}
 {{- end }}