diff --git a/.env.example b/.env.example index 2f0de0053..9965cc3b7 100644 --- a/.env.example +++ b/.env.example @@ -169,6 +169,10 @@ DATABASE_URL=postgresql://rhesis-user:your-secured-password@localhost:5432/rhesi # Development paths (adjust as needed) RHESIS_BASE_PATH=/path/to/your/rhesis/project/ +LOCAL_STORAGE_PATH=/tmp/rhesis-files # fallback for local development +# Optional: set these if you want to access GCS locally +# STORAGE_SERVICE_URI=gs://sources-rhesis-dev +# STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key' ##################################################################################### # 📚 QUICK START GUIDE diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cad8b51e6..df6fea7d6 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -88,6 +88,9 @@ jobs: SMTP_USER: ${{ secrets.SMTP_USER }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} FROM_EMAIL: ${{ secrets.FROM_EMAIL }} + STORAGE_SERVICE_URI: ${{ secrets.STORAGE_SERVICE_URI }} + STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }} + LOCAL_STORAGE_PATH: ${{ secrets.LOCAL_STORAGE_PATH }} services: postgres: diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index f4bec13ad..cde21f029 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -317,7 +317,9 @@ jobs: FROM_EMAIL=${{ secrets.FROM_EMAIL }}, BROKER_URL=${{ secrets.BROKER_URL }}, CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }}, - STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }} + STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }}, + STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }}, + LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }} EOF )" @@ -373,7 +375,9 @@ jobs: FROM_EMAIL=${{ secrets.FROM_EMAIL }}, BROKER_URL=${{ secrets.BROKER_URL }}, CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }}, - STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }} + STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }}, + STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }}, + LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }} EOF )" @@ -551,7 +555,9 @@ jobs: FROM_EMAIL=${{ secrets.FROM_EMAIL }}, BROKER_URL=${{ secrets.BROKER_URL }}, CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }}, - STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }} + STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }}, + STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }}, + LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }} EOF )" @@ -616,7 +622,9 @@ jobs: FROM_EMAIL=${{ secrets.FROM_EMAIL }}, BROKER_URL=${{ secrets.BROKER_URL }}, CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }}, - STORAGE_PROJECT_ID=${{ secrets.STORAGE_PROJECT_ID }} + STORAGE_SERVICE_URI=${{ secrets.STORAGE_SERVICE_URI }}, + STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }}, + LOCAL_STORAGE_PATH=${{ secrets.LOCAL_STORAGE_PATH }} EOF )" diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml index 598d3d164..f327e8f6c 100644 --- a/.github/workflows/worker.yml +++ b/.github/workflows/worker.yml @@ -288,7 +288,9 @@ jobs: --from-literal=BACKEND_ENV="${{ secrets.BACKEND_ENV }}" \ --from-literal=WORKER_ENV="${{ secrets.WORKER_ENV }}" \ --from-literal=FRONTEND_URL="${{ secrets.FRONTEND_URL }}" \ - --from-literal=STORAGE_PROJECT_ID="${{ secrets.STORAGE_PROJECT_ID }}" \ + --from-literal=STORAGE_SERVICE_URI="${{ secrets.STORAGE_SERVICE_URI }}" \ + --from-literal=STORAGE_SERVICE_ACCOUNT_KEY=${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }} \ + --from-literal=LOCAL_STORAGE_PATH="${{ secrets.LOCAL_STORAGE_PATH }}" \ --dry-run=client -o yaml | kubectl apply -f - - name: Deploy to GKE diff --git a/apps/worker/k8s/deployment.yaml b/apps/worker/k8s/deployment.yaml index b0b2157b9..54617bafa 100644 --- a/apps/worker/k8s/deployment.yaml +++ b/apps/worker/k8s/deployment.yaml @@ -177,6 +177,22 @@ spec: name: rhesis-worker-secrets key: worker_env optional: true + # Storage service configuration + - name: STORAGE_SERVICE_URI + valueFrom: + secretKeyRef: + name: rhesis-worker-secrets + key: STORAGE_SERVICE_URI + - name: STORAGE_SERVICE_ACCOUNT_KEY + valueFrom: + secretKeyRef: + name: rhesis-worker-secrets + key: STORAGE_SERVICE_ACCOUNT_KEY + - name: LOCAL_STORAGE_PATH + valueFrom: + secretKeyRef: + name: rhesis-worker-secrets + key: LOCAL_STORAGE_PATH ports: - containerPort: 8080 name: health diff --git a/infrastructure/config/service-secrets-config.sh.example b/infrastructure/config/service-secrets-config.sh.example index d4f526634..865894416 100755 --- a/infrastructure/config/service-secrets-config.sh.example +++ b/infrastructure/config/service-secrets-config.sh.example @@ -39,6 +39,10 @@ export DEV_FROM_EMAIL="engineering@example.com" export DEV_DEMO_USER_EMAIL="demo@rhesis.ai" export DEV_DEMO_USER_PASSWORD="PlatypusDemo!" export DEV_BACKEND_ENV="development" +export DEV_STORAGE_SERVICE_URI="gs://sources-rhesis-dev" +# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas +export DEV_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key' +export DEV_LOCAL_STORAGE_PATH="/tmp/rhesis-files" # Celery worker variables export DEV_BROKER_URL="rediss://:your-redis-auth-string@your-redis-host:6378/0?ssl_cert_reqs=CERT_NONE" export DEV_CELERY_RESULT_BACKEND="rediss://:your-redis-auth-string@your-redis-host:6378/1?ssl_cert_reqs=CERT_NONE" @@ -98,6 +102,10 @@ export STG_SMTP_PASSWORD="smtp-password" export STG_FROM_EMAIL="engineering@example.com" export STG_DEMO_USER_EMAIL="demo@rhesis.ai" export STG_DEMO_USER_PASSWORD="PlatypusDemo!" +export STG_STORAGE_SERVICE_URI="gs://sources-rhesis-stg" +# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas +export STG_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key' +export STG_LOCAL_STORAGE_PATH="/tmp/rhesis-files" # Celery worker variables export STG_WORKER_ENV="staging" export STG_BROKER_URL="rediss://:your-redis-auth-string@your-redis-host:6378/0?ssl_cert_reqs=CERT_NONE" @@ -157,6 +165,10 @@ export PRD_SMTP_PASSWORD="smtp-password" export PRD_FROM_EMAIL="engineering@example.com" export PRD_DEMO_USER_EMAIL="demo@rhesis.ai" export PRD_DEMO_USER_PASSWORD="PlatypusDemo!" +export PRD_STORAGE_SERVICE_URI="gs://sources-rhesis-prd" +# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas +export PRD_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key' +export PRD_LOCAL_STORAGE_PATH="/tmp/rhesis-files" # Celery worker variables export PRD_WORKER_ENV="production" export PRD_BROKER_URL="rediss://:your-redis-auth-string@your-redis-host:6378/0?ssl_cert_reqs=CERT_NONE" @@ -210,6 +222,10 @@ export TEST_SMTP_PASSWORD="smtp-password" export TEST_FROM_EMAIL="engineering@example.com" export TEST_DEMO_USER_EMAIL="demo@rhesis.ai" export TEST_DEMO_USER_PASSWORD="PlatypusDemo!" +export TEST_STORAGE_SERVICE_URI="gs://sources-rhesis-test" +# Note: STORAGE_SERVICE_ACCOUNT_KEY should be Base64 encoded JSON to avoid parsing issues with commas +export TEST_STORAGE_SERVICE_ACCOUNT_KEY='your-base64-service-account-key' +export TEST_LOCAL_STORAGE_PATH="/tmp/rhesis-files" export TEST_REDIS_URL="redis://localhost:6379" export TEST_BROKER_URL="redis://localhost:6379/0" export TEST_CELERY_RESULT_BACKEND="redis://localhost:6379/0" diff --git a/infrastructure/config/service-secrets-create.sh b/infrastructure/config/service-secrets-create.sh index 2362afa0f..e8baf6ee2 100755 --- a/infrastructure/config/service-secrets-create.sh +++ b/infrastructure/config/service-secrets-create.sh @@ -62,6 +62,9 @@ function show_usage() { echo " FROM_EMAIL Email address to use as sender" echo " DEMO_USER_EMAIL Demo user email address" echo " DEMO_USER_PASSWORD Demo user password" + echo " STORAGE_SERVICE_URI Google Cloud Storage bucket URI" + echo " STORAGE_SERVICE_ACCOUNT_KEY Google Cloud Storage service account key (Base64 encoded JSON)" + echo " LOCAL_STORAGE_PATH Local storage path for temporary files" echo "" echo " # Celery worker variables" echo " BROKER_URL Celery broker URL" @@ -69,6 +72,9 @@ function show_usage() { echo " CELERY_WORKER_CONCURRENCY Worker concurrency (number of processes)" echo " CELERY_WORKER_PREFETCH_MULTIPLIER Worker prefetch multiplier" echo " CELERY_WORKER_MAX_TASKS_PER_CHILD Max tasks per child process" + echo " STORAGE_SERVICE_URI Google Cloud Storage bucket URI" + echo " STORAGE_SERVICE_ACCOUNT_KEY Google Cloud Storage service account key (Base64 encoded JSON)" + echo " LOCAL_STORAGE_PATH Local storage path for temporary files" echo "" echo " # Frontend variables" echo " NEXTAUTH_URL NextAuth URL" @@ -218,6 +224,9 @@ SERVICE_VARS=( "FROM_EMAIL" "DEMO_USER_EMAIL" "DEMO_USER_PASSWORD" + "STORAGE_SERVICE_URI" + "STORAGE_SERVICE_ACCOUNT_KEY" + "LOCAL_STORAGE_PATH" # Celery worker variables "BROKER_URL" @@ -225,6 +234,9 @@ SERVICE_VARS=( "CELERY_WORKER_CONCURRENCY" "CELERY_WORKER_PREFETCH_MULTIPLIER" "CELERY_WORKER_MAX_TASKS_PER_CHILD" + "STORAGE_SERVICE_URI" + "STORAGE_SERVICE_ACCOUNT_KEY" + "LOCAL_STORAGE_PATH" # Frontend variables "NEXTAUTH_URL"