feat: Sentry integration (#20)

* feat: add sentry

* chore: replace black, isort, flake8 w/ ruff

* feat: sentry integration

* chore: remove test dataset

* ci: add sentry dsn to env

* ci: add deployment env var & commit SHA to pipelines

.env.example

@@ -1,7 +1,10 @@
+PYTHON_ENV=local
+DEPLOY_ENV=local
 DELTA_BEARER_TOKEN="$ openssl rand -base64 32"
 DELTA_SHARING_HOST="delta:8890"
 SECRET_KEY="$ tr -dc A-Za-z0-9 < /dev/urandom | head -c 64"
 APP_DOMAIN="localhost:5000"
+SENTRY_DSN=
 
 STORAGE_ACCESS_KEY="get from Azure portal"
 STORAGE_ACCOUNT_NAME="get from Azure portal"

.pre-commit-config.yaml

@@ -4,20 +4,12 @@ repos:
     hooks:
       - id: conventional-pre-commit
         stages: [ commit-msg ]
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.12
     hooks:
-      - id: black
-        language_version: python3
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        args: [ '--profile', 'black' ]
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:

Taskfile.yml

@@ -3,10 +3,12 @@ version: '3'
 dotenv:
   - .env
 
+env:
+  COMMIT_SHA:
+    sh: git rev-parse --short HEAD
+
 vars:
   PROJECT_NAME: giga-dataops_data-sharing
-  SHORT_SHA:
-    sh: git rev-parse --short HEAD
 
 tasks:
   default:
@@ -92,15 +94,15 @@ tasks:
     internal: true
     desc: Build Docker image
     cmds:
-      - docker build --tag {{.DATA_SHARING_PROXY_IMAGE_REPOSITORY}}:{{.SHORT_SHA}} -f proxy.prod.Dockerfile .
-      - docker build --tag {{.DATA_SHARING_DELTA_IMAGE_REPOSITORY}}:{{.SHORT_SHA}} -f delta.prod.Dockerfile .
+      - docker build --tag {{.DATA_SHARING_PROXY_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}} -f proxy.prod.Dockerfile .
+      - docker build --tag {{.DATA_SHARING_DELTA_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}} -f delta.prod.Dockerfile .
 
   push:
     internal: true
     desc: Push Docker image
     cmds:
-      - docker push {{.DATA_SHARING_PROXY_IMAGE_REPOSITORY}}:{{.SHORT_SHA}}
-      - docker push {{.DATA_SHARING_DELTA_IMAGE_REPOSITORY}}:{{.SHORT_SHA}}
+      - docker push {{.DATA_SHARING_PROXY_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}}
+      - docker push {{.DATA_SHARING_DELTA_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}}
 
   helm-up:
     desc: Install/upgrade Helm chart
@@ -114,12 +116,12 @@ tasks:
         helm upgrade --install data-sharing-delta ./helm/data-sharing-delta
         --namespace giga-data-sharing
         --set image.repository={{.DATA_SHARING_DELTA_IMAGE_REPOSITORY}}
-        --set image.tag={{.SHORT_SHA}}
+        --set image.tag={{.COMMIT_SHA}}
       - >
         helm upgrade --install data-sharing-proxy ./helm/data-sharing-proxy
         --namespace giga-data-sharing
         --set image.repository={{.DATA_SHARING_PROXY_IMAGE_REPOSITORY}}
-        --set image.tag={{.SHORT_SHA}}
+        --set image.tag={{.COMMIT_SHA}}
         --set service.type=LoadBalancer
 
   helm-down:

azure/azure-pipelines-dev.yaml

@@ -12,36 +12,4 @@ variables:
   - template: templates/variables.yaml
 
 stages:
-  - stage: BuildAndPushProxy
-    displayName: Build and push Data Sharing Proxy image
-    jobs:
-      - template: templates/build-and-push.yaml
-        parameters:
-          displayName: Data Sharing proxy
-          repository: giga-data-sharing-proxy
-          dockerfile: $(Build.SourcesDirectory)/proxy.prod.Dockerfile
-
-  - stage: BuildAndPushDeltaSharing
-    displayName: Build and push Delta Sharing server image
-    dependsOn: [ ]
-    jobs:
-      - template: templates/build-and-push.yaml
-        parameters:
-          displayName: Delta Sharing server
-          repository: giga-data-sharing-delta
-          dockerfile: $(Build.SourcesDirectory)/delta.prod.Dockerfile
-
-  - stage: CreateConfig
-    displayName: Create/update K8s configuration resources
-    dependsOn: [ ]
-    jobs:
-      - template: templates/create-config.yaml
-
-  - stage: Deploy
-    displayName: Deploy Data Sharing
-    dependsOn:
-      - BuildAndPushProxy
-      - BuildAndPushDeltaSharing
-      - CreateConfig
-    jobs:
-      - template: templates/helm-deploy.yaml
+  - template: templates/workflow.yaml

azure/azure-pipelines-prd.yaml

@@ -0,0 +1,15 @@
+trigger:
+  branches:
+    include:
+      - production
+
+pr: none
+
+pool:
+  vmImage: ubuntu-latest
+
+variables:
+  - template: templates/variables.yaml
+
+stages:
+  - template: templates/workflow.yaml

azure/azure-pipelines-stg.yaml

@@ -0,0 +1,15 @@
+trigger:
+  branches:
+    include:
+      - staging
+
+pr: none
+
+pool:
+  vmImage: ubuntu-latest
+
+variables:
+  - template: templates/variables.yaml
+
+stages:
+  - template: templates/workflow.yaml

azure/templates/create-config.yaml

@@ -44,3 +44,6 @@ jobs:
                     ADMIN_API_SECRET: "$(adminApiSecret)"
                     SECRET_KEY: "$(secretKey)"
                     APP_DOMAIN: "$(appDomain)"
+                    SENTRY_DSN: "$(sentryDsn)"
+                    COMMIT_SHA: "$(Build.SourceVersion)"
+                    DEPLOY_ENV: "$(deployEnv)"

azure/templates/helm-deploy.yaml

@@ -32,6 +32,6 @@ jobs:
                   --set image.repository="$(containerRegistryName).azurecr.io/giga-data-sharing-proxy"
                   --set image.tag="$(Build.SourceVersion)"
                   --set ingress.enabled=true
-                  --set ingress.hosts[0].host="$(appDomain)"
+                  --set ingress.hosts[0].host="io-datasharing-$(deployEnv).unitst.org"
                   --set ingress.hosts[0].paths[0].path="/"
                   --set ingress.hosts[0].paths[0].pathType=Exact

azure/templates/variables.yaml

@@ -13,4 +13,6 @@ variables:
   storageAccountName: $(STORAGE_ACCOUNT_NAME)
   storageContainerName: $(STORAGE_CONTAINER_NAME)
   storageContainerPath: $(STORAGE_CONTAINER_PATH)
+  sentryDsn: $(SENTRY_DSN)
+  deployEnv: $(DEPLOY_ENV)
   system.debug: true

azure/azure-pipelines-staging.yaml → azure/templates/workflow.yaml

@@ -1,21 +1,8 @@
-trigger:
-  branches:
-    include:
-      - staging
-
-pr: none
-
-pool:
-  vmImage: ubuntu-latest
-
-variables:
-  - template: templates/variables.yaml
-
 stages:
   - stage: BuildAndPushProxy
     displayName: Build and push Data Sharing Proxy image
     jobs:
-      - template: templates/build-and-push.yaml
+      - template: build-and-push.yaml
         parameters:
           displayName: Data Sharing proxy
           repository: giga-data-sharing-proxy
@@ -25,7 +12,7 @@ stages:
     displayName: Build and push Delta Sharing server image
     dependsOn: [ ]
     jobs:
-      - template: templates/build-and-push.yaml
+      - template: build-and-push.yaml
         parameters:
           displayName: Delta Sharing server
           repository: giga-data-sharing-delta
@@ -35,7 +22,7 @@ stages:
     displayName: Create/update K8s configuration resources
     dependsOn: [ ]
     jobs:
-      - template: templates/create-config.yaml
+      - template: create-config.yaml
 
   - stage: Deploy
     displayName: Deploy Data Sharing
@@ -44,4 +31,4 @@ stages:
       - BuildAndPushDeltaSharing
       - CreateConfig
     jobs:
-      - template: templates/helm-deploy.yaml
+      - template: helm-deploy.yaml

data_sharing/app.py

@@ -1,3 +1,4 @@
+import sentry_sdk
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import ORJSONResponse
@@ -6,6 +7,15 @@
 from data_sharing.routers import api_key, delta_sharing, role
 from data_sharing.settings import settings
 
+if settings.SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=settings.SENTRY_DSN,
+        traces_sample_rate=1.0,
+        profiles_sample_rate=1.0,
+        environment=settings.DEPLOY_ENV,
+        release=settings.COMMIT_SHA,
+    )
+
 app = FastAPI(
     title="Giga Data Sharing",
     version=__version__,

data_sharing/settings.py

@@ -11,7 +11,8 @@ class Config:
         env_file = ".env"
         extra = "ignore"
 
-    PYTHON_ENV: Literal["development", "staging", "production"] = "production"
+    PYTHON_ENV: Literal["local", "development", "staging", "production"] = "production"
+    DEPLOY_ENV: Literal["local", "dev", "stg", "prd"] = "local"
     BASE_DIR: Path = Path(__file__).parent.parent
     SECRET_KEY: str
     CORS_ALLOWED_ORIGINS: list[str] = ["*"]
@@ -24,10 +25,12 @@ class Config:
     DB_HOST: str
     APP_DOMAIN: str
     ADMIN_API_KEY: UUID4
+    SENTRY_DSN: str = ""
+    COMMIT_SHA: str = ""
 
     @property
     def IN_PRODUCTION(self) -> bool:
-        return self.PYTHON_ENV == "production"
+        return self.PYTHON_ENV != "local"
 
     @property
     def DATABASE_URL(self) -> str:

delta.dev.Dockerfile

@@ -1,24 +1,22 @@
-FROM maven:3.9-eclipse-temurin-11-alpine as base
+FROM maven:3.9-eclipse-temurin-11-alpine AS base
 
-FROM base as build
+FROM base AS build
 
 WORKDIR /tmp/custom-sas-provider
 
 COPY ./custom-sas-provider ./
 
 RUN mvn clean package
 
-FROM base as dev
+FROM base AS dev
 
 WORKDIR /tmp
 
-RUN wget https://github.com/delta-io/delta-sharing/releases/download/v1.0.2/delta-sharing-server-1.0.2.zip && \
-    unzip delta-sharing-server-1.0.2.zip
+RUN wget https://github.com/delta-io/delta-sharing/releases/download/v1.0.4/delta-sharing-server-1.0.4.zip && \
+    unzip delta-sharing-server-1.0.4.zip
 
 WORKDIR /app
 
-RUN cp -R /tmp/delta-sharing-server-1.0.2/* ./
-
-COPY --from=build /tmp/custom-sas-provider/target/custom-sas-provider-1.0-SNAPSHOT.jar ./lib/internal.giga.customSasProvider.custom-sas-provider-1.0-SNAPSHOT.jar
+RUN cp -R /tmp/delta-sharing-server-1.0.4/* ./
 
 CMD [ "./bin/delta-sharing-server", "--", "--config", "./conf/delta-sharing-server.yaml" ]