Merge branch 'scripts_with_post_request' into 'auth-spec'

Send command through url and use a websocket connection to keep reading the...

See merge request mistio/mist.api!994

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-slim-bullseye
+FROM python:3.11-slim-bullseye
 
 # Install libvirt which requires system dependencies.
 RUN apt update && \
@@ -20,7 +20,7 @@ RUN ln -s /influxdb-1.8.4-1/influxd /usr/local/bin/influxd && \
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir --upgrade setuptools && \
-    pip install libvirt-python==8.8.0 uwsgi==2.0.20 && \
+    pip install libvirt-python==8.8.0 uwsgi==2.0.21 && \
     pip install --no-cache-dir ipython ipdb flake8 pytest pytest-cov
 
 # Remove `-frozen` to build without strictly pinned dependencies.
@@ -63,4 +63,4 @@ ENV JS_BUILD=1 \
 
 
 RUN echo "{\"sha\":\"$VERSION_SHA\",\"name\":\"$VERSION_NAME\",\"repo\":\"$VERSION_REPO\",\"modified\":false}" \
-        > /mist-version.json
+    > /mist-version.json

migrations/0019-cleanup-keys-libvirt-locations.py

@@ -21,7 +21,7 @@ def cleanup_libvirt_cloud_locations():
     """
     from mist.api.models import Machine
     from mist.api.clouds.models import CloudLocation, LibvirtCloud
-    libvirt_cloud_ids = [l.id for l in LibvirtCloud.objects(
+    libvirt_cloud_ids = [loc.id for loc in LibvirtCloud.objects(
         deleted=None).only('id')]
 
     for loc in CloudLocation.objects(cloud__in=libvirt_cloud_ids):

migrations/0041-remove-control-plane-cost.py

@@ -23,7 +23,7 @@ def remove_control_plane_costs():
         '$unset': {"cost.control_plane_monthly": 1}
     })
     print(f'{res.modified_count} machines were modified.')
-    print(f'Removing control_plane_monthly from clusters ...')
+    print('Removing control_plane_monthly from clusters ...')
     res = db_clusters.update_many({}, {
         '$unset': {"cost.control_plane_monthly": 1}
     })

requirements-frozen.txt

@@ -3,8 +3,9 @@
 ## direct dependecies and their dependencies and so on. This ensures that
 ## builds wont start failing just because of a new release of some dependency.
 
-amqp==2.6.1 # 5.1.1
+amqp==5.1.1
 apscheduler==3.9.1
+asgiref==3.5.2
 asn1crypto==1.5.1
 atomicwrites==1.4.1
 attrs==22.1.0
@@ -22,8 +23,7 @@ cryptography==38.0.1
 dateparser==1.1.1
 decorator==4.4.2
 dnspython==1.16.0
-elasticsearch==6.8.0
-elasticsearch-tornado==2.0.9
+elasticsearch[async]==7.10.1
 flake8==5.0.4
 future==0.18.2
 funcsigs==1.0.2
@@ -32,15 +32,15 @@ greenlet==1.1.3.post0
 idna==2.10
 ipaddress==1.0.23
 ipdb==0.13.9
-ipython==7.34.0
+ipython==8.7.0
 ipython-genutils==0.2.0
 iso8601==0.1.16
 jedi==0.18.1
 Jinja2==2.11.3
 jsonpatch==1.32
 jsonpickle==2.2.0
 jsonpointer==2.3
-kombu==4.6.11
+kombu==5.2.4
 Logbook==1.5.3
 lxml==4.9.1
 Mako==1.2.3
@@ -63,7 +63,7 @@ pathlib2==2.3.7.post1
 pbr==5.10.0
 pexpect==4.8.0
 pickleshare==0.7.5
-pika==0.12.0 # 1.3.0
+pika==1.3.1
 pingparsing==1.4.0
 pluggy==0.13.1
 pretty==0.1
@@ -99,20 +99,19 @@ sentry-sdk==1.9.10
 simplegeneric==0.8.1
 singledispatch==3.7.0
 six==1.16.0
-sockjs-tornado==1.0.6
+sockjs-tornado==1.0.7
 subprocrunner==1.6.0
-tornado==5.1.1
+tornado==6.2
 troposphere==3.2.2
-#tornado-profile==1.2.0
 traitlets==5.4.0
 translationstring==1.4
 typepy==1.3.0
 typing_extensions==4.4.0
 urllib3==1.26.12
 uwsgidecorators==1.1.0
 venusian==1.2.0
-vine==1.3.0
+vine==5.0.0
 wcwidth==0.2.5
 WebOb==1.8.7
 websocket-client==1.4.1
-yappi==1.3.6
+yappi==1.4.0

requirements.txt

@@ -6,15 +6,15 @@
 ## ensures that the build won't break because of a new release of some
 ## dependency.
 
-amqp<3.0
+amqp
 apscheduler
+asgiref
 beautifulsoup4
 boto3
 dnspython
 dateparser
 dramatiq
 elasticsearch
-elasticsearch_tornado
 flake8
 future
 gevent
@@ -47,6 +47,7 @@ pytest
 python3-openid
 pyvmomi
 requests
+rstream
 s3cmd
 scp
 sendgrid-python
@@ -57,4 +58,4 @@ tornado
 troposphere
 #tornado_profile
 uwsgidecorators
-websocket-client
+websocket-client

src/mist/api/amqp_tornado.py

@@ -9,8 +9,7 @@
 import logging
 
 import pika
-from pika import adapters
-
+import pika.adapters.tornado_connection
 
 log = logging.getLogger(__name__)
 
@@ -61,8 +60,8 @@ def connect(self):
 
         """
         log.info('Connecting to %s', self.amqp_url)
-        return adapters.TornadoConnection(pika.URLParameters(self.amqp_url),
-                                          self.on_connection_open)
+        return pika.adapters.tornado_connection.TornadoConnection(
+            pika.URLParameters(self.amqp_url), self.on_connection_open)
 
     def close_connection(self):
         """This method closes the connection to RabbitMQ."""
@@ -77,7 +76,7 @@ def add_on_connection_close_callback(self):
         log.debug('Adding connection close callback')
         self._connection.add_on_close_callback(self.on_connection_closed)
 
-    def on_connection_closed(self, connection, reply_code, reply_text):
+    def on_connection_closed(self, connection, reply_code, reply_text=''):
         """This method is invoked by pika when the connection to RabbitMQ is
         closed unexpectedly. Since it is unexpected, we will reconnect to
         RabbitMQ if it disconnects.
@@ -125,7 +124,7 @@ def add_on_channel_close_callback(self):
         log.debug('Adding channel close callback')
         self._channel.add_on_close_callback(self.on_channel_closed)
 
-    def on_channel_closed(self, channel, reply_code, reply_text):
+    def on_channel_closed(self, channel, reply_code, reply_text=''):
         """Invoked by pika when RabbitMQ unexpectedly closes the channel.
         Channels are usually closed if you attempt to do something that
         violates the protocol, such as re-declare an exchange or queue with
@@ -164,10 +163,10 @@ def setup_exchange(self, exchange_name):
 
         """
         log.debug('Declaring exchange %s', exchange_name)
-        self._channel.exchange_declare(self.on_exchange_declareok,
-                                       exchange_name,
+        self._channel.exchange_declare(exchange_name,
                                        self.exchange_type,
-                                       **self.exchange_kwargs)
+                                       callback=self.on_exchange_declareok,
+                                       arguments=self.exchange_kwargs)
 
     def on_exchange_declareok(self, unused_frame):
         """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
@@ -189,8 +188,9 @@ def setup_queue(self, queue_name):
 
         """
         log.debug('Declaring queue %s', queue_name)
-        self._channel.queue_declare(self.on_queue_declareok, queue_name,
-                                    **self.queue_kwargs)
+        self._channel.queue_declare(queue_name,
+                                    callback=self.on_queue_declareok,
+                                    arguments=self.queue_kwargs)
 
     def on_queue_declareok(self, method_frame):
         """Method invoked by pika when the Queue.Declare RPC call made in
@@ -204,8 +204,8 @@ def on_queue_declareok(self, method_frame):
         """
         log.debug('Binding %s to %s with %s',
                   self.exchange, self.queue, self.routing_key)
-        self._channel.queue_bind(self.on_bindok, self.queue,
-                                 self.exchange, self.routing_key)
+        self._channel.queue_bind(self.queue, self.exchange, self.routing_key,
+                                 callback=self.on_bindok)
 
     def add_on_cancel_callback(self):
         """Add a callback that will be invoked if RabbitMQ cancels the consumer
@@ -276,7 +276,8 @@ def stop_consuming(self):
         """
         if self._channel:
             log.info('Sending a Basic.Cancel RPC command to RabbitMQ')
-            self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
+            self._channel.basic_cancel(
+                consumer_tag=self._consumer_tag, callback=self.on_cancelok)
 
     def start_consuming(self):
         """This method sets up the consumer by first calling
@@ -290,9 +291,9 @@ def start_consuming(self):
         """
         log.debug('Issuing consumer related RPC commands')
         self.add_on_cancel_callback()
-        self._consumer_tag = self._channel.basic_consume(self.on_message,
-                                                         self.queue,
-                                                         no_ack=not self.ack)
+        self._consumer_tag = self._channel.basic_consume(self.queue,
+                                                         self.on_message,
+                                                         auto_ack=not self.ack)
 
     def on_bindok(self, unused_frame):
         """Invoked by pika when the Queue.Bind method has completed. At this