Feat/Configure Storage Service Variables

.env.example

@@ -169,6 +169,10 @@ DATABASE_URL=postgresql://rhesis-user:your-secured-password@localhost:5432/rhesi
 
 # Development paths (adjust as needed)
 RHESIS_BASE_PATH=/path/to/your/rhesis/project/
+LOCAL_STORAGE_PATH=/tmp/rhesis-files   # fallback for local development
+# Optional: set these if you want to access GCS locally
+# STORAGE_SERVICE_URI=gs://sources-rhesis-dev
+# STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key'
 
 #####################################################################################
 # 📚 QUICK START GUIDE

.github/workflows/backend-test.yml

@@ -88,6 +88,9 @@ jobs:
       SMTP_USER: ${{ secrets.SMTP_USER }}
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
       FROM_EMAIL: ${{ secrets.FROM_EMAIL }}
+      STORAGE_SERVICE_URI: ${{ secrets.STORAGE_SERVICE_URI }}
+      STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }}
+      LOCAL_STORAGE_PATH: ${{ secrets.LOCAL_STORAGE_PATH }}
 
     services:
       postgres:

.github/workflows/backend.yml

@@ -317,7 +317,9 @@ jobs:
           FROM_EMAIL=${{ secrets.FROM_EMAIL }},
           BROKER_URL=${{ secrets.BROKER_URL }},
           CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }},
-          STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }}
+          STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }},
+          STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }},
+          LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }}
           EOF
           )"
 
@@ -373,7 +375,9 @@ jobs:
           FROM_EMAIL=${{ secrets.FROM_EMAIL }},
           BROKER_URL=${{ secrets.BROKER_URL }},
           CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }},
-          STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }}
+          STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }},
+          STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }},
+          LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }}
           EOF
           )"
 
@@ -551,7 +555,9 @@ jobs:
           FROM_EMAIL=${{ secrets.FROM_EMAIL }},
           BROKER_URL=${{ secrets.BROKER_URL }},
           CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }},
-          STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }}
+          STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }},
+          STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }},
+          LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }}
           EOF
           )"
 
@@ -616,7 +622,9 @@ jobs:
           FROM_EMAIL=${{ secrets.FROM_EMAIL }},
           BROKER_URL=${{ secrets.BROKER_URL }},
           CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }},
-          STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }}
+          STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }},
+          STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }},
+          LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }}
           EOF
           )"
 

.github/workflows/worker.yml

@@ -288,7 +288,9 @@ jobs:
             --from-literal=BACKEND_ENV="${{ secrets.BACKEND_ENV }}" \
             --from-literal=WORKER_ENV="${{ secrets.WORKER_ENV }}" \
             --from-literal=FRONTEND_URL="${{ secrets.FRONTEND_URL }}" \
-            --from-literal=STORAGE_PROJECT_ID="${{ secrets.STORAGE_PROJECT_ID }}" \
+            --from-literal=STORAGE_SERVICE_URI="${{ secrets.STORAGE_SERVICE_URI }}" \
+            --from-literal=STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }} \
+            --from-literal=LOCAL_STORAGE_PATH="${{ secrets.LOCAL_STORAGE_PATH }}" \
             --dry-run=client -o yaml | kubectl apply -f -
 
       - name: Deploy to GKE

apps/worker/k8s/deployment.yaml

@@ -177,6 +177,22 @@ spec:
               name: rhesis-worker-secrets
               key: worker_env
               optional: true
+        # Storage service configuration
+        - name: STORAGE_SERVICE_URI
+          valueFrom:
+            secretKeyRef:
+              name: rhesis-worker-secrets
+              key: STORAGE_SERVICE_URI
+        - name: STORAGE_SERVICE_ACCOUNT_KEY
+          valueFrom:
+            secretKeyRef:
+              name: rhesis-worker-secrets
+              key: STORAGE_SERVICE_ACCOUNT_KEY
+        - name: LOCAL_STORAGE_PATH
+          valueFrom:
+            secretKeyRef:
+              name: rhesis-worker-secrets
+              key: LOCAL_STORAGE_PATH
         ports:
         - containerPort: 8080
           name: health

infrastructure/config/service-secrets-config.sh.example

@@ -39,6 +39,10 @@ export DEV_FROM_EMAIL="[email protected]"
 export DEV_DEMO_USER_EMAIL="[email protected]"
 export DEV_DEMO_USER_PASSWORD="PlatypusDemo!"
 export DEV_BACKEND_ENV="development"
+export DEV_STORAGE_SERVICE_URI="gs://sources-rhesis-dev"
+# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas
+export DEV_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key'
+export DEV_LOCAL_STORAGE_PATH="/tmp/rhesis-files"
 # Celery worker variables
 export DEV_BROKER_URL="rediss://:your-redis-auth-string@your-redis-host:6378/0?ssl_cert_reqs=CERT_NONE"
 export DEV_CELERY_RESULT_BACKEND="rediss://:your-redis-auth-string@your-redis-host:6378/1?ssl_cert_reqs=CERT_NONE"
@@ -98,6 +102,10 @@ export STG_SMTP_PASSWORD="smtp-password"
 export STG_FROM_EMAIL="[email protected]"
 export STG_DEMO_USER_EMAIL="[email protected]"
 export STG_DEMO_USER_PASSWORD="PlatypusDemo!"
+export STG_STORAGE_SERVICE_URI="gs://sources-rhesis-stg"
+# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas
+export STG_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key'
+export STG_LOCAL_STORAGE_PATH="/tmp/rhesis-files"
 # Celery worker variables
 export STG_WORKER_ENV="staging"
 export STG_BROKER_URL="rediss://:your-redis-auth-string@your-redis-host:6378/0?ssl_cert_reqs=CERT_NONE"
@@ -157,6 +165,10 @@ export PRD_SMTP_PASSWORD="smtp-password"
 export PRD_FROM_EMAIL="[email protected]"
 export PRD_DEMO_USER_EMAIL="[email protected]"
 export PRD_DEMO_USER_PASSWORD="PlatypusDemo!"
+export PRD_STORAGE_SERVICE_URI="gs://sources-rhesis-prd"
+# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas
+export PRD_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key'
+export PRD_LOCAL_STORAGE_PATH="/tmp/rhesis-files"
 # Celery worker variables
 export PRD_WORKER_ENV="production"
 export PRD_BROKER_URL="rediss://:your-redis-auth-string@your-redis-host:6378/0?ssl_cert_reqs=CERT_NONE"
@@ -210,6 +222,10 @@ export TEST_SMTP_PASSWORD="smtp-password"
 export TEST_FROM_EMAIL="[email protected]"
 export TEST_DEMO_USER_EMAIL="[email protected]"
 export TEST_DEMO_USER_PASSWORD="PlatypusDemo!"
+export TEST_STORAGE_SERVICE_URI="gs://sources-rhesis-test"
+# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas
+export TEST_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key'
+export TEST_LOCAL_STORAGE_PATH="/tmp/rhesis-files"
 export TEST_REDIS_URL="redis://localhost:6379"
 export TEST_BROKER_URL="redis://localhost:6379/0"
 export TEST_CELERY_RESULT_BACKEND="redis://localhost:6379/0"

infrastructure/config/service-secrets-create.sh

@@ -62,13 +62,19 @@ function show_usage() {
   echo "  FROM_EMAIL                    Email address to use as sender"
   echo "  DEMO_USER_EMAIL               Demo user email address"
   echo "  DEMO_USER_PASSWORD            Demo user password"
+  echo "  STORAGE_SERVICE_URI           Google Cloud Storage bucket URI"
+  echo "  STORAGE_SERVICE_ACCOUNT_KEY   Google Cloud Storage service account key (Base64 encoded JSON)"
+  echo "  LOCAL_STORAGE_PATH            Local storage path for temporary files"
   echo ""
   echo "  # Celery worker variables"
   echo "  BROKER_URL                    Celery broker URL"
   echo "  CELERY_RESULT_BACKEND         Celery result backend URL"
   echo "  CELERY_WORKER_CONCURRENCY     Worker concurrency (number of processes)"
   echo "  CELERY_WORKER_PREFETCH_MULTIPLIER Worker prefetch multiplier"
   echo "  CELERY_WORKER_MAX_TASKS_PER_CHILD Max tasks per child process"
+  echo "  STORAGE_SERVICE_URI           Google Cloud Storage bucket URI"
+  echo "  STORAGE_SERVICE_ACCOUNT_KEY   Google Cloud Storage service account key (Base64 encoded JSON)"
+  echo "  LOCAL_STORAGE_PATH            Local storage path for temporary files"
   echo ""
   echo "  # Frontend variables"
   echo "  NEXTAUTH_URL                  NextAuth URL"
@@ -218,13 +224,19 @@ SERVICE_VARS=(
   "FROM_EMAIL"
   "DEMO_USER_EMAIL"
   "DEMO_USER_PASSWORD"
+  "STORAGE_SERVICE_URI"
+  "STORAGE_SERVICE_ACCOUNT_KEY"
+  "LOCAL_STORAGE_PATH"
 
   # Celery worker variables
   "BROKER_URL"
   "CELERY_RESULT_BACKEND"
   "CELERY_WORKER_CONCURRENCY"
   "CELERY_WORKER_PREFETCH_MULTIPLIER"
   "CELERY_WORKER_MAX_TASKS_PER_CHILD"
+  "STORAGE_SERVICE_URI"
+  "STORAGE_SERVICE_ACCOUNT_KEY"
+  "LOCAL_STORAGE_PATH"
 
   # Frontend variables
   "NEXTAUTH_URL"