Merge branch 'auth-spec' into 'staging'

Allow AuthTokens to support multiple orgs

See merge request mistio/mist.api!954

.gitlab-ci.yml

@@ -12,6 +12,7 @@ build_mist_image:
   before_script:
     # Configure registries.
     - |
+        rm ~/.docker/config.json
         export REGISTRIES=""
 
         # Login to gitlab docker registry.
@@ -62,15 +63,14 @@ build_mist_image:
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
   script:
-    - docker pull mist/python3
-    - docker build --rm -t gcr.io/mist-ops/mist:$CI_COMMIT_SHA --build-arg API_VERSION_SHA=$CI_COMMIT_SHA --build-arg API_VERSION_NAME=$CI_COMMIT_REF_NAME --build-arg CI_API_V4_URL=$CI_API_V4_URL .
-    - docker tag gcr.io/mist-ops/mist:$CI_COMMIT_SHA gcr.io/mist-ops/mist:$CI_COMMIT_REF_SLUG
-    - docker tag gcr.io/mist-ops/mist:$CI_COMMIT_SHA mist/mist:$CI_COMMIT_SHA
-    - docker tag gcr.io/mist-ops/mist:$CI_COMMIT_REF_SLUG mist/mist:$CI_COMMIT_REF_SLUG
-    - docker push gcr.io/mist-ops/mist:$CI_COMMIT_SHA
-    - docker push gcr.io/mist-ops/mist:$CI_COMMIT_REF_SLUG
-    - docker push mist/mist:$CI_COMMIT_SHA
-    - docker push mist/mist:$CI_COMMIT_REF_SLUG
+    - docker build --rm -t registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA --build-arg API_VERSION_SHA=$CI_COMMIT_SHA --build-arg API_VERSION_NAME=$CI_COMMIT_REF_NAME --build-arg CI_API_V4_URL=$CI_API_V4_URL .
+    - docker tag registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_REF_SLUG
+    - docker tag registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA mist/api:$CI_COMMIT_SHA
+    - docker tag registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA mist/api:$CI_COMMIT_REF_SLUG
+    - docker push registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA
+    - docker push registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_REF_SLUG
+    - docker push mist/api:$CI_COMMIT_SHA
+    - docker push mist/api:$CI_COMMIT_REF_SLUG
   tags:
     - builder
   dependencies: []
@@ -79,7 +79,7 @@ build_mist_image:
 #################### GENERATE_API_SPEC STAGE ####################
 generate_api_spec:
   stage: generate_api_spec
-  image: gcr.io/mist-ops/mist:$CI_COMMIT_SHA
+  image: registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA
   before_script:
     - cd /mist.api
   script:
@@ -90,7 +90,7 @@ generate_api_spec:
 #################### TEST STAGE ####################
 flake8:
   stage: test
-  image: gcr.io/mist-ops/mist:$CI_COMMIT_SHA
+  image: registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
   before_script:
@@ -100,7 +100,7 @@ flake8:
 
 uniq:
   stage: test
-  image: gcr.io/mist-ops/mist:$CI_COMMIT_SHA
+  image: registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
   before_script:
@@ -110,7 +110,7 @@ uniq:
 
 unit_tests:
   stage: test
-  image: gcr.io/mist-ops/mist:$CI_COMMIT_SHA
+  image: registry.ops.mist.io/mistio/mist.api:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
   before_script:

Dockerfile

@@ -1,10 +1,10 @@
-FROM python:3.7-slim-buster
+FROM python:3.11-slim-bullseye
 
 # Install libvirt which requires system dependencies.
 RUN apt update && \
     apt install -y git build-essential g++ gcc cargo gnupg ca-certificates \
-    libssl-dev libffi-dev libvirt-dev libxml2-dev libxslt-dev zlib1g-dev \
-    mongo-tools libmemcached-dev procps netcat wget curl jq inetutils-ping && \
+    libssl-dev libffi-dev libvirt-dev libxml2-dev libxslt-dev zlib1g-dev vim \
+    libmemcached-dev procps netcat wget curl jq inetutils-ping && \
     rm -rf /var/lib/apt/lists/*
 
 RUN wget https://dl.influxdata.com/influxdb/releases/influxdb-1.8.4-static_linux_amd64.tar.gz && \
@@ -15,12 +15,12 @@ RUN ln -s /influxdb-1.8.4-1/influxd /usr/local/bin/influxd && \
     ln -s /usr/bin/python3 /usr/bin/python
 
 # Download VictoriaMetrics promql middleware .so file
-ARG CI_API_V4_URL
-RUN wget -O promql_middleware.so  `curl "${CI_API_V4_URL}/projects/126/releases" | jq -r .[0].assets.links[0].url`
+# ARG CI_API_V4_URL
+# RUN wget -O promql_middleware.so  `curl "${CI_API_V4_URL}/projects/126/releases" | jq -r .[0].assets.links[0].url`
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir --upgrade setuptools && \
-    pip install libvirt-python==7.2.0 uwsgi==2.0.19.1 && \
+    pip install libvirt-python==8.8.0 uwsgi==2.0.21 && \
     pip install --no-cache-dir ipython ipdb flake8 pytest pytest-cov
 
 # Remove `-frozen` to build without strictly pinned dependencies.
@@ -63,4 +63,4 @@ ENV JS_BUILD=1 \
 
 
 RUN echo "{\"sha\":\"$VERSION_SHA\",\"name\":\"$VERSION_NAME\",\"repo\":\"$VERSION_REPO\",\"modified\":false}" \
-        > /mist-version.json
+    > /mist-version.json

bin/init-portal-admin

@@ -52,7 +52,7 @@ def main():
             return
         api_token = ApiToken()
         api_token.name = "auto-created"
-        api_token.org = org
+        api_token.orgs = [org]
         api_token.set_user(user)
         api_token.token = token
         try:

migrations/0019-cleanup-keys-libvirt-locations.py

@@ -21,7 +21,7 @@ def cleanup_libvirt_cloud_locations():
     """
     from mist.api.models import Machine
     from mist.api.clouds.models import CloudLocation, LibvirtCloud
-    libvirt_cloud_ids = [l.id for l in LibvirtCloud.objects(
+    libvirt_cloud_ids = [loc.id for loc in LibvirtCloud.objects(
         deleted=None).only('id')]
 
     for loc in CloudLocation.objects(cloud__in=libvirt_cloud_ids):

migrations/0041-remove-control-plane-cost.py

@@ -23,7 +23,7 @@ def remove_control_plane_costs():
         '$unset': {"cost.control_plane_monthly": 1}
     })
     print(f'{res.modified_count} machines were modified.')
-    print(f'Removing control_plane_monthly from clusters ...')
+    print('Removing control_plane_monthly from clusters ...')
     res = db_clusters.update_many({}, {
         '$unset': {"cost.control_plane_monthly": 1}
     })

requirements-frozen.txt

@@ -3,116 +3,115 @@
 ## direct dependecies and their dependencies and so on. This ensures that
 ## builds wont start failing just because of a new release of some dependency.
 
-amqp==2.6.1
-apscheduler==3.7.0
-asn1crypto==1.3.0
-atomicwrites==1.3.0
-attrs==19.3.0
-bcrypt==3.1.7
-billiard==3.6.3.0
-beautifulsoup4==4.9.3
-boto3==1.21.2
-certifi==2019.11.28
-cffi==1.14.0
-Chameleon==3.6.2
+amqp==5.1.1
+apscheduler==3.9.1
+asgiref==3.5.2
+asn1crypto==1.5.1
+atomicwrites==1.4.1
+attrs==22.1.0
+bcrypt==3.2.2
+billiard==3.6.4.0
+beautifulsoup4==4.11.1
+boto3==1.24.91
+certifi==2022.9.24
+cffi==1.15.1
+Chameleon==3.10.1
 chardet==3.0.4
 configparser==3.8.1
 coverage==4.5.4
-cryptography==3.4.7
-dateparser==0.7.4
+cryptography==38.0.1
+dateparser==1.1.1
 decorator==4.4.2
 dnspython==1.16.0
-elasticsearch==6.8.0
-elasticsearch-tornado==2.0.9
-flake8==3.7.9
+elasticsearch[async]==7.10.1
+flake8==5.0.4
 future==0.18.2
 funcsigs==1.0.2
-gevent==1.4.0
-greenlet==0.4.15
-idna==2.9
+gevent==22.10.1
+greenlet==1.1.3.post0
+idna==2.10
 ipaddress==1.0.23
-ipdb==0.13.2
-ipython==7.16.3
+ipdb==0.13.9
+ipython==8.7.0
 ipython-genutils==0.2.0
-iso8601==0.1.12
-jedi==0.17.2
+iso8601==0.1.16
+jedi==0.18.1
 Jinja2==2.11.3
-jsonpatch==1.25
-jsonpickle==1.3
-jsonpointer==2.0
-kombu==4.6.11
+jsonpatch==1.32
+jsonpickle==2.2.0
+jsonpointer==2.3
+kombu==5.2.4
 Logbook==1.5.3
 lxml==4.9.1
-Mako==1.1.2
+Mako==1.2.3
 MarkupSafe==1.1.1
-mbstrdecoder==0.8.4
-mccabe==0.6.1
+mbstrdecoder==1.1.0
+mccabe==0.7.0
 mock==4.0.2
-mongoengine==0.23.1
-mongomock==3.19.0
-more-itertools==8.2.0
-msgfy==0.0.7
+mongoengine==0.24.2
+mongomock==3.23.0
+more-itertools==8.14.0
+msgfy==0.2.0
 names==0.3.0
-netaddr==0.7.19
-parse==1.15.0
-passlib==1.7.2
-Paste==3.4.0
-PasteDeploy==2.1.0
-PasteScript==3.2.0
-pathlib2==2.3.5
-pbr==5.4.4
+netaddr==0.7.20
+parse==1.19.0
+passlib==1.7.4
+Paste==3.5.2
+PasteDeploy==2.1.1
+PasteScript==3.2.1
+pathlib2==2.3.7.post1
+pbr==5.10.0
 pexpect==4.8.0
 pickleshare==0.7.5
-pika==0.12.0
-pingparsing==1.0.1
+pika==1.3.1
+pingparsing==1.4.0
 pluggy==0.13.1
 pretty==0.1
-prompt-toolkit==3.0.4
-ptyprocess==0.6.0
-py==1.10.0
+prompt-toolkit==3.0.31
+ptyprocess==0.7.0
+py==1.11.0
 pyasn1==0.4.8
-pycodestyle==2.5.0
-pycparser==2.20
-pycryptodome==3.9.7
-pyflakes==2.1.1
-Pygments==2.8.1
-pymongo==3.10.1
-PyNaCl==1.3.0
-pyOpenSSL==19.1.0
-pyparsing==2.4.6
+pycodestyle==2.9.1
+pycryptodome==3.15.0
+pyflakes==2.5.0
+Pygments==2.13.0
+pymongo==3.12.3
+PyNaCl==1.5.0
+pyOpenSSL==22.1.0
+pyparsing==2.4.7
 pyramid==1.6.5
 pyramid_chameleon==0.3
 pytest==5.4.1
 pytest-cov==2.8.1
 python-dateutil==2.8.1
 python-http-client==3.2.6
 python3-openid==3.1.0
-pytz==2019.3
-pyvmomi==6.7.3
+pytz==2022.4
+pyvmomi==7.0.3
 PyYAML==5.4.1
 repoze.lru==0.7
 requests==2.27.1
 scandir==1.10.0
-sendgrid==6.2.0
+sendgrid==6.9.7
 sentinels==1.0.0
 sentry-dramatiq==0.3.2
-sentry-sdk==1.4.3
+sentry-sdk==1.9.10
 simplegeneric==0.8.1
-singledispatch==3.4.0.3
-six==1.14.0
-sockjs-tornado==1.0.6
-subprocrunner==1.2.0
-tornado==5.1.1
+singledispatch==3.7.0
+six==1.16.0
+sockjs-tornado==1.0.7
+subprocrunner==1.6.0
+tornado==6.2
 troposphere==3.2.2
-#tornado-profile==1.2.0
-traitlets==4.3.3
-translationstring==1.3
-typepy==0.6.6
-urllib3==1.26.5
+traitlets==5.4.0
+translationstring==1.4
+typepy==1.3.0
+typing_extensions==4.4.0
+urllib3==1.26.12
 uwsgidecorators==1.1.0
 venusian==1.2.0
-vine==1.3.0
-wcwidth==0.1.9
-WebOb==1.8.6
-websocket-client==0.57.0
-yappi==1.2.3
+vine==5.0.0
+wcwidth==0.2.5
+WebOb==1.8.7
+websocket-client==1.4.1
+yappi==1.4.0

requirements.txt

@@ -6,15 +6,15 @@
 ## ensures that the build won't break because of a new release of some
 ## dependency.
 
-amqp<3.0
+amqp
 apscheduler
+asgiref
 beautifulsoup4
 boto3
 dnspython
 dateparser
 dramatiq
 elasticsearch
-elasticsearch_tornado
 flake8
 future
 gevent
@@ -47,6 +47,7 @@ pytest
 python3-openid
 pyvmomi
 requests
+rstream
 s3cmd
 scp
 sendgrid-python
@@ -57,4 +58,4 @@ tornado
 troposphere
 #tornado_profile
 uwsgidecorators
-websocket-client
+websocket-client