Feature/opentelemetry (#1467)

* uninstall opencensus * working solution without duplicate logging * add logs as traces to Application Insights * adds test and logging of userid + name to requests * add formatting * ignore devcontainer setup * fix last flake8 comments * fixes isort * fix isort correctly * fix whitespace for isort * add spdx headers * fix mypy * remove todo comment * mypy synced baseline * remove unnecessary exception view
Amsterdam · Sep 24, 2024 · 24156d8 · 24156d8
1 parent ace7b42
commit 24156d8
Show file tree

Hide file tree

Showing 9 changed files with 514 additions and 292 deletions.
diff --git a/.gitignore b/.gitignore
@@ -181,6 +181,7 @@ Global/NetBeans.gitignore
 .env/
 .mypy_cache/
 notes/
+.devcontainer
 
 # bind mount should be ignored
 dwh_media/

diff --git a/app/mypy-baseline.txt b/app/mypy-baseline.txt
@@ -1,10 +1,9 @@
 signals/admin/oidc/backends.py:0: error: Skipping analyzing "mozilla_django_oidc.auth": module is installed, but missing library stubs or py.typed marker  [import-untyped]
 signals/admin/oidc/backends.py:0: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-signals/settings.py:0: error: Dict entry 1 has incompatible type "str": "str | None"; expected "str": "str"  [dict-item]
 signals/apps/services/domain/pdf.py:0: error: Skipping analyzing "weasyprint": module is installed, but missing library stubs or py.typed marker  [import-untyped]
-signals/apps/signals/models/reporter.py:0: error: Skipping analyzing "django_fsm": module is installed, but missing library stubs or py.typed marker  [import-untyped]
 signals/apps/signals/models/category.py:0: error: Skipping analyzing "django_extensions.db.fields": module is installed, but missing library stubs or py.typed marker  [import-untyped]
 signals/apps/signals/models/category.py:0: error: Skipping analyzing "rest_framework_extensions.settings": module is installed, but missing library stubs or py.typed marker  [import-untyped]
+signals/apps/signals/models/reporter.py:0: error: Skipping analyzing "django_fsm": module is installed, but missing library stubs or py.typed marker  [import-untyped]
 signals/apps/signals/models/status_message.py:0: error: Need type annotation for "categories"  [var-annotated]
 signals/apps/email_integrations/admin.py:0: error: Skipping analyzing "markdownx.widgets": module is installed, but missing library stubs or py.typed marker  [import-untyped]
 signals/apps/search/rest_framework/serializers.py:0: error: Skipping analyzing "elasticsearch_dsl.response": module is installed, but missing library stubs or py.typed marker  [import-untyped]

diff --git a/app/requirements/requirements.in b/app/requirements/requirements.in
@@ -1,3 +1,4 @@
+azure-monitor-opentelemetry-exporter
 azure-storage-blob
 beautifulsoup4
 Celery
@@ -25,11 +26,9 @@ jwcrypto
 lxml
 mozilla-django-oidc
 networkx
-opencensus-ext-azure
-opencensus-ext-django
-opencensus-ext-logging
-opencensus-ext-postgresql
-opencensus-ext-requests
+opentelemetry-instrumentation-django
+opentelemetry-instrumentation-psycopg2
+opentelemetry-sdk
 Pillow
 psycopg2-binary
 pypdf

diff --git a/app/requirements/requirements.txt b/app/requirements/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 amqp==5.2.0
     # via kombu
@@ -20,11 +20,11 @@ autopep8==2.3.1
     # via django-silk
 azure-core==1.30.2
     # via
-    #   azure-identity
+    #   azure-monitor-opentelemetry-exporter
     #   azure-storage-blob
-    #   opencensus-ext-azure
-azure-identity==1.17.1
-    # via opencensus-ext-azure
+    #   msrest
+azure-monitor-opentelemetry-exporter==1.0.0b28
+    # via -r requirements.in
 azure-storage-blob==12.21.0
     # via -r requirements.in
 beautifulsoup4==4.12.3
@@ -33,8 +33,6 @@ beautifulsoup4==4.12.3
     #   o365
 billiard==4.2.0
     # via celery
-cachetools==5.4.0
-    # via google-auth
 cairocffi==1.7.1
     # via
     #   cairosvg
@@ -51,6 +49,7 @@ celery==5.4.0
 certifi==2024.7.4
     # via
     #   elasticsearch
+    #   msrest
     #   requests
 cffi==1.16.0
     # via
@@ -75,13 +74,10 @@ cron-descriptor==1.4.3
     # via django-celery-beat
 cryptography==43.0.1
     # via
-    #   azure-identity
     #   azure-storage-blob
     #   josepy
     #   jwcrypto
     #   mozilla-django-oidc
-    #   msal
-    #   pyjwt
     #   pyopenssl
 cssselect2==0.7.0
     # via
@@ -91,6 +87,10 @@ defusedxml==0.7.1
     # via
     #   cairosvg
     #   djangorestframework-xml
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-semantic-conventions
 django==4.2.15
     # via
     #   -r requirements.in
@@ -109,7 +109,6 @@ django==4.2.15
     #   drf-spectacular
     #   drf-spectacular-sidecar
     #   mozilla-django-oidc
-    #   opencensus-ext-django
 django-appconf==1.0.6
     # via django-celery-email
 django-celery-beat==2.6.0
@@ -164,14 +163,10 @@ elasticsearch==7.17.9
     # via elasticsearch-dsl
 elasticsearch-dsl==7.4.1
     # via -r requirements.in
+fixedint==0.1.6
+    # via azure-monitor-opentelemetry-exporter
 flower==2.0.1
     # via -r requirements.in
-google-api-core==2.19.1
-    # via opencensus
-google-auth==2.32.0
-    # via google-api-core
-googleapis-common-protos==1.63.2
-    # via google-api-core
 gprof2dot==2024.6.6
     # via django-silk
 html5lib==1.1
@@ -180,10 +175,14 @@ humanize==4.10.0
     # via flower
 idna==3.7
     # via requests
+importlib-metadata==8.4.0
+    # via opentelemetry-api
 inflection==0.5.1
     # via drf-spectacular
 isodate==0.6.1
-    # via azure-storage-blob
+    # via
+    #   azure-storage-blob
+    #   msrest
 josepy==1.14.0
     # via mozilla-django-oidc
 jsonschema==4.23.0
@@ -202,76 +201,70 @@ markdown==3.6
     # via django-markdownx
 mozilla-django-oidc==4.0.1
     # via -r requirements.in
-msal==1.30.0
-    # via
-    #   azure-identity
-    #   msal-extensions
-msal-extensions==1.2.0
-    # via azure-identity
+msrest==0.7.1
+    # via azure-monitor-opentelemetry-exporter
 networkx==3.3
     # via -r requirements.in
 o365==2.0.36
     # via django-o365mail
 oauthlib==3.2.2
     # via requests-oauthlib
-opencensus==0.11.4
-    # via
-    #   opencensus-ext-azure
-    #   opencensus-ext-django
-    #   opencensus-ext-logging
-    #   opencensus-ext-postgresql
-    #   opencensus-ext-requests
-opencensus-context==0.1.3
-    # via opencensus
-opencensus-ext-azure==1.1.13
-    # via -r requirements.in
-opencensus-ext-django==0.8.0
+opentelemetry-api==1.27.0
+    # via
+    #   azure-monitor-opentelemetry-exporter
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-dbapi
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-psycopg2
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-instrumentation==0.48b0
+    # via
+    #   opentelemetry-instrumentation-dbapi
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-psycopg2
+    #   opentelemetry-instrumentation-wsgi
+opentelemetry-instrumentation-dbapi==0.48b0
+    # via opentelemetry-instrumentation-psycopg2
+opentelemetry-instrumentation-django==0.48b0
     # via -r requirements.in
-opencensus-ext-logging==0.1.1
-    # via -r requirements.in
-opencensus-ext-postgresql==0.1.3
-    # via -r requirements.in
-opencensus-ext-requests==0.8.0
+opentelemetry-instrumentation-psycopg2==0.48b0
     # via -r requirements.in
+opentelemetry-instrumentation-wsgi==0.48b0
+    # via opentelemetry-instrumentation-django
+opentelemetry-sdk==1.27.0
+    # via
+    #   -r requirements.in
+    #   azure-monitor-opentelemetry-exporter
+opentelemetry-semantic-conventions==0.48b0
+    # via
+    #   opentelemetry-instrumentation-dbapi
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.48b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
 pillow==10.4.0
     # via
     #   -r requirements.in
     #   cairosvg
     #   django-markdownx
     #   weasyprint
-portalocker==2.10.1
-    # via msal-extensions
 prometheus-client==0.20.0
     # via flower
 prompt-toolkit==3.0.47
     # via click-repl
-proto-plus==1.24.0
-    # via google-api-core
-protobuf==5.27.2
-    # via
-    #   google-api-core
-    #   googleapis-common-protos
-    #   proto-plus
-psutil==6.0.0
-    # via opencensus-ext-azure
+psutil==5.9.8
+    # via azure-monitor-opentelemetry-exporter
 psycopg2-binary==2.9.9
-    # via
-    #   -r requirements.in
-    #   opencensus-ext-postgresql
-pyasn1==0.6.0
-    # via
-    #   pyasn1-modules
-    #   rsa
-pyasn1-modules==0.4.0
-    # via google-auth
+    # via -r requirements.in
 pycodestyle==2.12.0
     # via autopep8
 pycparser==2.22
     # via cffi
-pyjwt[crypto]==2.8.0
-    # via
-    #   msal
-    #   pyjwt
 pyopenssl==24.2.1
     # via josepy
 pypdf==4.3.1
@@ -301,28 +294,24 @@ requests==2.32.3
     # via
     #   -r requirements.in
     #   azure-core
-    #   google-api-core
     #   mozilla-django-oidc
-    #   msal
+    #   msrest
     #   o365
-    #   opencensus-ext-azure
-    #   opencensus-ext-requests
     #   requests-oauthlib
 requests-oauthlib==2.0.0
-    # via o365
+    # via
+    #   msrest
+    #   o365
 rpds-py==0.19.1
     # via
     #   jsonschema
     #   referencing
-rsa==4.9
-    # via google-auth
 six==1.16.0
     # via
     #   azure-core
     #   elasticsearch-dsl
     #   html5lib
     #   isodate
-    #   opencensus
     #   python-dateutil
 soupsieve==2.5
     # via beautifulsoup4
@@ -342,9 +331,9 @@ tornado==6.4.1
 typing-extensions==4.12.2
     # via
     #   azure-core
-    #   azure-identity
     #   azure-storage-blob
     #   jwcrypto
+    #   opentelemetry-sdk
 tzdata==2024.1
     # via
     #   celery
@@ -375,7 +364,12 @@ webencodings==0.5.1
     #   html5lib
     #   tinycss2
 wrapt==1.16.0
-    # via opencensus-ext-requests
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-dbapi
+zipp==3.20.1
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools