Document Engine 1.4.1 and asset storage fallback fixes (#16)

PSPDFKit · Jul 31, 2024 · 0e6a281 · 0e6a281
1 parent 20c1cfe
commit 0e6a281
Show file tree

Hide file tree

Showing 12 changed files with 189 additions and 43 deletions.
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml
@@ -45,4 +45,4 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           ct install --target-branch ${{ github.event.repository.default_branch }} \
-             --helm-extra-args "--timeout 600s"
+             --helm-extra-args "--timeout 300s"
diff --git a/charts/document-engine/CHANGELOG.md b/charts/document-engine/CHANGELOG.md
@@ -1,38 +1,56 @@
 # Changelog
 
 - [Changelog](#changelog)
-  - [2.8.0](#280)
+  - [2.9.0](#290)
     - [Added](#added)
     - [Changed](#changed)
     - [Fixed](#fixed)
-  - [2.7.3](#273)
+  - [2.8.0](#280)
+    - [Added](#added-1)
     - [Changed](#changed-1)
     - [Fixed](#fixed-1)
-  - [2.7.2](#272)
+  - [2.7.3](#273)
+    - [Changed](#changed-2)
     - [Fixed](#fixed-2)
+  - [2.7.2](#272)
+    - [Fixed](#fixed-3)
   - [2.7.0](#270)
-    - [Changed](#changed-2)
-  - [2.6.2](#262)
-    - [Added](#added-1)
     - [Changed](#changed-3)
-  - [2.6.0](#260)
+  - [2.6.2](#262)
     - [Added](#added-2)
-  - [2.4.0](#240)
+    - [Changed](#changed-4)
+  - [2.6.0](#260)
     - [Added](#added-3)
-  - [2.3.0](#230)
+  - [2.4.0](#240)
     - [Added](#added-4)
-  - [2.2.0](#220)
+  - [2.3.0](#230)
     - [Added](#added-5)
+  - [2.2.0](#220)
+    - [Added](#added-6)
   - [2.1.0](#210)
-    - [Changed](#changed-4)
-  - [2.0.0](#200)
     - [Changed](#changed-5)
+  - [2.0.0](#200)
+    - [Changed](#changed-6)
+
+## 2.9.0
+
+### Added
+
+* Azure blob storage support.
+* Introduced `pspdfkit.signingTrustConfigMaps` for ConfigMaps to mount to `/certificate-stores/`
+
+### Changed
+
+* [Document Engine 1.4.1](https://pspdfkit.com/changelog/document-engine/#1.4.1)
+
+### Fixed
+
+* Asset storage fallback.
 
 ## 2.8.0
 
 ### Added
 
-* [Document Engine 1.4.0](https://pspdfkit.com/changelog/document-engine/#1.4.0)
 * Support for OpenTelemetry traces, enabled by setting `pspdfkit.observability.opentelemetry.enabled` to `true`. 
   * Unless the collector is placed as a sidecar and receives by grpc at port `4317`, other parameters are also necessary. 
   * Please note: standard OpenTelemetry environment variables are used, and the following values are just convenience wrappers, therefore other configuration approaches (e.g. setting variables through mutations or post build patches) will also work.
@@ -46,6 +64,7 @@
 
 ### Changed
 
+* [Document Engine 1.4.0](https://pspdfkit.com/changelog/document-engine/#1.4.0)
 * Changed `pspdfkit.storage.enableMigrationJobs` to `pspdfkit.storage.databaseMigrationJob.enabled`.
 * Renamed `.Values.pspdfkit.storage.redis.sentinels` to `.Values.pspdfkit.storage.redis.sentinel`.
 * Slight refinement of trust information parameters: all files from `pspdfkit.trustConfigMaps` are now mounted to `/certificate-stores-custom/` to avoid confusion with `/certificate-stores/` which services for document signature validation certificates.

diff --git a/charts/document-engine/Chart.yaml b/charts/document-engine/Chart.yaml
@@ -3,8 +3,8 @@ name: document-engine
 description: PSPDFKit Document Engine
 icon: data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTAwMCIgaGVpZ2h0PSIxMDAwIiB2aWV3Qm94PSIwIDAgMTAwMCAxMDAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgo8cGF0aCBkPSJNMzQyLjU2NyA2NTguNjEyTDMyOS40OTYgNjY1LjkxNUwzMzguNzk5IDY1NC4yODZDMzk5LjMxIDU3OS44NDIgNDMzLjAyNiA0ODcuMjExIDQzNC41MjggMzkxLjI4OEw0MzQuNzYgMzc2LjMxTDQ0MC4xMSAzOTAuMjY1QzQ3NC40MDUgNDc5LjkzMiA1MzcuODc4IDU1NS40NDggNjIwLjMxIDYwNC42NTRMNjMzLjE0OCA2MTIuMzI5TDYxOC4zNTYgNjA5Ljk1N0M1MjMuNTcxIDU5NC43NTYgNDI2LjQyOCA2MTEuODk0IDM0Mi41NjcgNjU4LjYxMloiIGZpbGw9IiMxOTBEOTQiLz4KPHBhdGggZD0iTTU1NC41MTkgNTU2LjM2N0M0NzMuMzk2IDQ0Mi4wMzMgNDg3LjMwNCAzNzYuODY1IDUwOS44NjQgMjg5Ljc4OEM1MzYuNjExIDE4Ni41MjQgNDczLjM1IDg0LjA1MDUgMzYyLjU5NyAxMTQuNDI1QzMxMy41MjMgMTI3LjkxNCAyOTYuMTczIDE2MS4zMTIgMjkwLjAzMyAxODQuNzU2QzI1OS45ODQgMjk3Ljc4OSA0MzguMjMxIDQ2Ni4xNzQgNTU0LjUxOSA1NTYuMzY3WiIgZmlsbD0iIzVFNUNFQiIvPgo8cGF0aCBkPSJNMjE3LjEwMyA2NDQuMTg2QzExNC4zMDQgNjcyLjY1NCA1Ny4xODM1IDc3OC43NTUgMTM4Ljg2NSA4NTkuNDEzQzE3NS4wNTMgODk1LjEzNyAyMTIuNjM4IDg5My40NjIgMjM2LjAzNSA4ODcuMzIyQzM0OC45MjggODU2Ljc2MiA0MDUuNjMgNjE4LjE4NCA0MjUuNjMyIDQ3Mi40MDVDMzY3LjE2MiA1OTkuNjI1IDMwMy45OTQgNjIwLjE4NCAyMTcuMTAzIDY0NC4xODZaIiBmaWxsPSIjNUU1Q0VCIi8+CjxwYXRoIGQ9Ik04NzEuNDggNTgyLjY1MUM3ODguNTQzIDUwMC4xNzkgNTUzLjU5NCA1NzAuMzI0IDQxNy4zNSA2MjUuOTFDNTU2Ljg5NiA2MTIuODM5IDYwNi4zODkgNjU3LjQ0OCA2NzAuNTMzIDcyMC41MjJDNzQ2LjU4NiA3OTUuMzE5IDg2Ni45NjggNzkxLjczNyA4OTYuMDQgNjgwLjY1OUM5MDguNjkyIDYzMS40NDUgODg4LjQ1OCA1OTkuNzIyIDg3MS40OCA1ODIuNjUxWiIgZmlsbD0iIzVFNUNFQiIvPgo8L3N2Zz4K
 type: application
-version: 2.8.1
-appVersion: "1.4.0"
+version: 2.9.0
+appVersion: "1.4.1"
 
 maintainers:
   - name: PSPDFKit

diff --git a/charts/document-engine/ci/2-with-db-values.yaml b/charts/document-engine/ci/2-with-db-values.yaml
@@ -19,7 +19,7 @@ postgresql:
   primary:
     persistence:
       enabled: true
-      size: 1Gi
+      size: 512Mi
       storageClass: standard
       accessModes:
         - ReadWriteOnce

diff --git a/charts/document-engine/ci/3-with-db-s3-redis-values.yaml b/charts/document-engine/ci/3-with-db-s3-redis-values.yaml
@@ -7,6 +7,8 @@ pspdfkit:
       podLabels:
         do: "that"
     assetStorageBackend: s3
+    enableAssetStorageFallback: true
+    enableAssetStorageFallbackPostgres: true
     postgres:
       enabled: true
       host: postgresql
@@ -46,7 +48,7 @@ postgresql:
   primary:
     persistence:
       enabled: true
-      size: 1Gi
+      size: 512Mi
       storageClass: standard
       accessModes:
         - ReadWriteOnce

diff --git a/charts/document-engine/ci/4-with-db-azure-values.yaml b/charts/document-engine/ci/4-with-db-azure-values.yaml
@@ -0,0 +1,49 @@
+pspdfkit:
+  storage:
+    databaseMigrationJob:
+      enabled: false
+    assetStorageBackend: azure
+    enableAssetStorageFallback: true
+    enableAssetStorageFallbackPostgres: true
+    postgres:
+      enabled: true
+      host: postgresql
+      port: 5432
+      username: postgres
+    azure:
+      # This is "well known" storage account for Azure Storage Emulator
+      # See https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string#configure-a-connection-string-for-azurite
+      accountName: devstoreaccount1
+      accountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+      container: "document-engine-assets"
+      apiUrl: "http://127.0.0.1:10000/devstoreaccount1"
+
+# Azurite as sidecar
+sidecars:
+  - name: azurite
+    image: mcr.microsoft.com/azure-storage/azurite:latest
+    imagePullPolicy: Always
+    ports:
+      - containerPort: 10000
+        name: blob
+        protocol: TCP
+
+postgresql:
+  enabled: true
+  fullnameOverride: postgresql
+  nameOverride: postgresql
+  primary:
+    persistence:
+      enabled: true
+      size: 512Mi
+      storageClass: standard
+      accessModes:
+        - ReadWriteOnce
+  volumePermissions:
+    enabled: true
+
+minio:
+  enabled: false
+
+redis:
+  enabled: false
diff --git a/charts/document-engine/ci/README.md b/charts/document-engine/ci/README.md
@@ -3,3 +3,4 @@
 * `1-no-storage-values.yaml` — Document Engine with no storage, processing only
 * `2-with-db-values.yaml` — Basic configuration with PostgreSQL database
 * `3-with-db-s3-redis-values.yaml` — PostgreSQL, MinIO as S3 asset storage backend and Redis for rendering cache
+* `4-with-db-azure-values.yaml` — PostgreSQL and Azurite to test Azure Blob storage
diff --git a/charts/document-engine/templates/_helpers.tpl b/charts/document-engine/templates/_helpers.tpl
@@ -99,7 +99,8 @@ License secret name
 {{- end -}}
 
 {{- define "document-engine.license.available" -}}
-  {{- if or .Values.pspdfkit.license.activationKey .Values.pspdfkit.license.externalSecret.name -}}
+  {{- if or .Values.pspdfkit.license.activationKey 
+            .Values.pspdfkit.license.externalSecret.name -}}
     {{- true -}}
   {{- else -}}
     {{- false -}}
@@ -187,7 +188,8 @@ Database secrets
   {{- end -}}
 {{- end -}}
 {{- define "document-engine.storage.postgres.createSecret" -}}
-  {{- if and .Values.pspdfkit.storage.postgres.enabled (not .Values.pspdfkit.storage.postgres.externalSecretName) -}}
+  {{- if and .Values.pspdfkit.storage.postgres.enabled 
+             (not .Values.pspdfkit.storage.postgres.externalSecretName) -}}
     {{- true -}}
   {{- else -}}
     {{- false -}}
@@ -202,7 +204,8 @@ Database secrets
   {{- end -}}
 {{- end -}}
 {{- define "document-engine.storage.postgres.createAdminSecret" -}}
-  {{- if and .Values.pspdfkit.storage.postgres.enabled (not .Values.pspdfkit.storage.postgres.externalAdminSecretName) -}}
+  {{- if and .Values.pspdfkit.storage.postgres.enabled 
+             (not .Values.pspdfkit.storage.postgres.externalAdminSecretName) -}}
     {{- true -}}
   {{- else -}}
     {{- false -}}
@@ -217,7 +220,8 @@ Database secrets
   {{- end -}}
 {{- end -}}
 {{- define "document-engine.storage.redis.createSecret" -}}
-  {{- if and .Values.pspdfkit.storage.redis.enabled (not .Values.pspdfkit.storage.redis.externalSecretName) -}}
+  {{- if and .Values.pspdfkit.storage.redis.enabled 
+             (not .Values.pspdfkit.storage.redis.externalSecretName) -}}
     {{- true -}}
   {{- else -}}
     {{- false -}}
@@ -228,21 +232,18 @@ Database secrets
 Object storage parameters
 */}}
 {{- define "document-engine.storage.s3.enabled" -}}
-  {{- if .Values.pspdfkit.storage.assetStorageBackend -}}
-    {{- with .Values.pspdfkit.storage.assetStorageBackend -}}
-      {{- if eq . "s3" -}}
-        {{- true -}}
-      {{- else -}}
-        {{- false -}}
-      {{- end -}}
-    {{- end -}}
+  {{- if or (eq .Values.pspdfkit.storage.assetStorageBackend "s3")
+            (and .Values.pspdfkit.storage.enableAssetStorageFallback
+                 .Values.pspdfkit.storage.enableAssetStorageFallbackS3 ) -}}
+    {{- true -}}
   {{- else -}}
     {{- false -}}
   {{- end -}}
 {{- end -}}
 
 {{- define "document-engine.storage.s3.createSecret" -}}
-  {{- if and (eq (include "document-engine.storage.s3.enabled" .) "true") (not .Values.pspdfkit.storage.s3.externalSecretName) -}}
+  {{- if and (eq (include "document-engine.storage.s3.enabled" .) "true") 
+             (not .Values.pspdfkit.storage.s3.externalSecretName) -}}
     {{- true -}}
   {{- else -}}
     {{- false -}}
@@ -257,6 +258,34 @@ Object storage parameters
   {{- end -}}
 {{- end -}}
 
+{{- define "document-engine.storage.azure.enabled" -}}
+  {{- if or (eq .Values.pspdfkit.storage.assetStorageBackend "azure")
+            (and .Values.pspdfkit.storage.enableAssetStorageFallback
+                 .Values.pspdfkit.storage.enableAssetStorageFallbackAzure ) -}}
+    {{- true -}}
+  {{- else -}}
+    {{- false -}}
+  {{- end -}}
+{{- end -}}
+
+
+{{- define "document-engine.storage.azure.createSecret" -}}
+  {{- if and (eq (include "document-engine.storage.azure.enabled" .) "true") 
+             (not .Values.pspdfkit.storage.azure.externalSecretName) -}}
+    {{- true -}}
+  {{- else -}}
+    {{- false -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "document-engine.storage.azure.secret.name" -}}
+  {{- if (eq (include "document-engine.storage.azure.createSecret" .) "true") -}}
+    {{- printf "%s-s3" (include "document-engine.fullname" .) -}}
+  {{- else -}}
+    {{- .Values.pspdfkit.storage.azure.externalSecretName -}}
+  {{- end -}}
+{{- end -}}
+
 {{/*
 Jobs
 */}}

diff --git a/charts/document-engine/templates/config.ConfigMap.yaml b/charts/document-engine/templates/config.ConfigMap.yaml
@@ -31,17 +31,17 @@ data:
 {{- end }}
 {{- with .Values.pspdfkit.storage }}
   ASSET_STORAGE_BACKEND: {{ default "built-in" .assetStorageBackend | quote }}
-{{- if .enableAssetStorageFallback }}
-  ENABLE_ASSET_STORAGE_FALLBACK: "false"
-  ENABLE_ASSET_STORAGE_FALLBACK_POSTGRES: "false"
-  ENABLE_ASSET_STORAGE_FALLBACK_S3: "false"
-  ENABLE_ASSET_STORAGE_FALLBACK_AZURE: "false"
-{{- else }}
+{{-   if .enableAssetStorageFallback }}
   ENABLE_ASSET_STORAGE_FALLBACK: {{ .enableAssetStorageFallback | quote }}
   ENABLE_ASSET_STORAGE_FALLBACK_POSTGRES: {{ .enableAssetStorageFallbackPostgres | quote }}
   ENABLE_ASSET_STORAGE_FALLBACK_S3: {{ .enableAssetStorageFallbackS3 | quote }}
   ENABLE_ASSET_STORAGE_FALLBACK_AZURE: {{ .enableAssetStorageFallbackAzure | quote }}
-{{- end }}
+{{-   else }}
+  ENABLE_ASSET_STORAGE_FALLBACK: "false"
+  ENABLE_ASSET_STORAGE_FALLBACK_POSTGRES: "false"
+  ENABLE_ASSET_STORAGE_FALLBACK_S3: "false"
+  ENABLE_ASSET_STORAGE_FALLBACK_AZURE: "false"
+{{-   end }}
   USE_REDIS_CACHE: {{ .redis.enabled | quote }}
   USE_REDIS_TTL_FOR_PRERENDERING: {{ .redis.useTtlForPrerendering | quote }}
   REDIS_TTL: {{ .redis.ttlSeconds | int | quote }}

diff --git a/charts/document-engine/templates/config.storage-azure.Secret.yaml b/charts/document-engine/templates/config.storage-azure.Secret.yaml
@@ -0,0 +1,23 @@
+{{- if (eq (include "document-engine.storage.azure.createSecret" .) "true") }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "document-engine.storage.azure.secret.name" . }}
+  labels:
+    {{- include "document-engine.labels" . | nindent 4 }}
+data:
+{{- with .Values.pspdfkit.storage }}
+{{-   if eq .assetStorageBackend "azure" }}
+  AZURE_STORAGE_DEFAULT_CONTAINER: {{ .azure.container | b64enc | quote }}
+{{-     if .azure.connectionString }}
+  AZURE_STORAGE_ACCOUNT_CONNECTION_STRING: {{ .azure.connectionString | b64enc | quote }}
+{{-     else if and .azure.accountName .azure.accountKey }}
+  AZURE_STORAGE_ACCOUNT_NAME: {{ .azure.accountName | b64enc | quote }}
+  AZURE_STORAGE_ACCOUNT_KEY: {{ .azure.accountKey | b64enc | quote }}
+{{-     end }}
+{{-     if .azure.apiUrl }}
+  AZURE_STORAGE_API_URL: {{ .azure.apiUrl | b64enc | quote }}
+{{-     end }}
+{{-   end }}
+{{- end }}
+{{- end }}
diff --git a/charts/document-engine/templates/deployment.yaml b/charts/document-engine/templates/deployment.yaml
@@ -47,6 +47,9 @@ spec:
       {{- if (include "document-engine.storage.s3.createSecret" .) }}
         checksum/s3-secret: {{ include (print $.Template.BasePath "/config.storage-s3.Secret.yaml") . | sha256sum }}
       {{- end }}
+      {{- if (include "document-engine.storage.azure.createSecret" .) }}
+        checksum/azure-secret: {{ include (print $.Template.BasePath "/config.storage-azure.Secret.yaml") . | sha256sum }}
+      {{- end }}
       {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -84,6 +87,10 @@ spec:
             - secretRef:
                 name: {{ include "document-engine.storage.s3.secret.name" . }}
           {{- end }}
+          {{- if eq (include "document-engine.storage.azure.enabled" .) "true" }}
+            - secretRef:
+                name: {{ include "document-engine.storage.azure.secret.name" . }}
+          {{- end }}
           {{- with .Values.extraEnvFrom }}
             {{- toYaml . | nindent 12 }}
           {{- end }}