Revert "Move from psycopg 2 to 3"

This reverts commit a602dee.

Makefile

@@ -1,5 +1,5 @@
 prefix := /var/www/workflow
-PYTHON_VERSION:=3.12
+PYTHON_VERSION:=3.11
 
 # these are defined here because I couldn't figure out how to evaluate the commands
 # during target execution rather then when make parses the file
@@ -47,7 +47,7 @@ check: ## Check python dependencies
 	echo CONDA_PREFIX=${CONDA_PREFIX}
 
 	@python -c "import django" || echo "\nERROR: Django is not installed: www.djangoproject.com\n"
-	@python -c "import psycopg" || echo "\nWARNING: psycopg is not installed: http://initd.org/psycopg\n"
+	@python -c "import psycopg2" || echo "\nWARNING: psycopg2 is not installed: http://initd.org/psycopg\n"
 	@python -c "import stomp" || echo "\nERROR: stomp.py is not installed: http://code.google.com/p/stomppy\n"
 
 wheel/dasmon: ## create or update python wheel for service "dasmon". Clean up build/ first

README.rst

@@ -20,7 +20,7 @@ Dependencies:
  * `stomp <https://github.com/jasonrbriggs/stomp.py>`_
  * `django <https://www.djangoproject.com/>`_
  * `MySQLdb <https://sourceforge.net/projects/mysql-python/>`_ if using MySQL
- * `psycopg <https://www.psycopg.org/>`_ if using PostgreSQL
+ * `psycopg2 <https://www.psycopg.org/>`_ if using PostgreSQL
 
 It consists of 3 applications (Workflow Manager, Web Monitor, and DASMON Listener) which are deployed via docker compose.
 

conda_environment.yml

@@ -3,14 +3,14 @@ channels:
   - conda-forge
   - oncat
 dependencies:
-  - python=3.12
+  - python=3.11
   - django=4.2
   - gunicorn
   - httplib2
   - make
   - oauthlib
   - postgresql
-  - psycopg>=3.1.8
+  - psycopg2-binary=2.9.3
   - psutil
   - python-ldap>=3.1
   - requests-oauthlib

src/dasmon_app/dasmon_listener/settings.py

@@ -5,7 +5,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": os.environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": os.environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": os.environ.get("DATABASE_PASS"),  # Not used with sqlite3.

src/webmon_app/docker-entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-MANAGE_PY_WEBMON="/opt/conda/lib/python3.12/site-packages/reporting/manage.py"
+MANAGE_PY_WEBMON="/opt/conda/lib/python3.11/site-packages/reporting/manage.py"
 
 # wait for postgress to be available
 until PGPASSWORD=${DATABASE_PASS} psql -h "${DATABASE_HOST}" -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -c '\q'; do

src/webmon_app/reporting/reporting_app/settings/base.py

@@ -137,7 +137,7 @@ def validate_ldap_settings(server_uri, user_dn_template):
 # The DB settings are defined the same as in the workflow manager
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": environ.get("DATABASE_PASS"),  # Not used with sqlite3.

src/workflow_app/workflow/database/settings.py

@@ -3,7 +3,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": os.environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": os.environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": os.environ.get("DATABASE_PASS"),  # Not used with sqlite3.

tests/conftest.py

@@ -1,6 +1,6 @@
 # third-party imports
 from dotenv import dotenv_values
-import psycopg
+import psycopg2
 import pytest
 import stomp
 
@@ -67,8 +67,8 @@ def db_connection():
     """Database connection with config from env files"""
     config = {**dotenv_values(".env"), **dotenv_values(".env.ci")}
     assert config
-    conn = psycopg.connect(
-        dbname=config["DATABASE_NAME"],
+    conn = psycopg2.connect(
+        database=config["DATABASE_NAME"],
         user=config["DATABASE_USER"],
         password=config["DATABASE_PASS"],
         port=config["DATABASE_PORT"],

tests/test_PostProcessWorkflow.py

@@ -1,4 +1,4 @@
-import psycopg
+import psycopg2
 import requests
 import time
 from django.conf import settings
@@ -13,8 +13,8 @@ class TestPostProcessingWorkflow:
     def setup_class(cls):
         config = {**dotenv_values(".env"), **dotenv_values(".env.ci")}
         assert config
-        cls.conn = psycopg.connect(
-            dbname=config["DATABASE_NAME"],
+        cls.conn = psycopg2.connect(
+            database=config["DATABASE_NAME"],
             user=config["DATABASE_USER"],
             password=config["DATABASE_PASS"],
             port=config["DATABASE_PORT"],

tests/test_ReductionSetupPageView.py

@@ -1,6 +1,6 @@
 import os
 import time
-import psycopg
+import psycopg2
 import pytest
 import requests
 import subprocess
@@ -49,18 +49,16 @@ def getReductionScriptContents(self):
         ).decode()
 
     def initReductionGroup(self, conn, cursor):
-        cursor.execute("SELECT id FROM report_instrument where name = %s;", ("arcs",))
-        inst_id = cursor.fetchone()[0]
-        cursor.execute("SELECT * from reduction_reductionproperty WHERE instrument_id = %s;", (inst_id,))
+        cursor.execute("SELECT * from reduction_reductionproperty WHERE instrument_id = 3;")
         if cursor.fetchone() is None:
             timestamp = datetime.datetime.now()
             cursor.execute(
                 "INSERT INTO reduction_reductionproperty (instrument_id, key, value, timestamp) VALUES(%s, %s, %s, %s)",
-                (inst_id, "grouping", "/SNS/ARCS/shared/autoreduce/ARCS_2X1_grouping.xml", timestamp),
+                (3, "grouping", "/SNS/ARCS/shared/autoreduce/ARCS_2X1_grouping.xml", timestamp),
             )
             conn.commit()
 
-        cursor.execute("SELECT * from reduction_choice WHERE instrument_id = %s;", (inst_id,))
+        cursor.execute("SELECT * from reduction_choice WHERE instrument_id = 3;")
         if cursor.fetchone() is None:
             cursor.execute("SELECT * FROM reduction_reductionproperty WHERE key = 'grouping';")
             props = cursor.fetchone()
@@ -110,8 +108,8 @@ def testReduction(self, instrument_scientist_client):
 
         assert "this is a template" not in self.getReductionScriptContents()
 
-        conn = psycopg.connect(
-            dbname="workflow",
+        conn = psycopg2.connect(
+            database="workflow",
             user="workflow",
             password="workflow",
             port="5432",

tests/test_SMS_messages.py

@@ -36,18 +36,16 @@ def clear_run(self, conn, run_number):
             return
 
         cursor.execute(
-            "SELECT id FROM report_datarun WHERE instrument_id_id = %s AND run_number = %s;", (inst_id[0], run_number)
+            "SELECT id FROM report_datarun WHERE instrument_id_id = %s AND run_number = %s;", (inst_id, run_number)
         )
         run_id = cursor.fetchone()
         if run_id is None:
             return
 
-        run_id = run_id[0]
-
         db_utils.clear_previous_runstatus(conn, run_id)
-        cursor.execute("DELETE FROM report_workflowsummary WHERE run_id_id = %s;", (run_id,))
-        cursor.execute("DELETE FROM report_instrumentstatus WHERE last_run_id_id = %s;", (run_id,))
-        cursor.execute("DELETE FROM report_datarun WHERE id = %s;", (run_id,))
+        cursor.execute("DELETE FROM report_workflowsummary WHERE run_id_id = %s;", run_id)
+        cursor.execute("DELETE FROM report_instrumentstatus WHERE last_run_id_id = %s;", run_id)
+        cursor.execute("DELETE FROM report_datarun WHERE id = %s;", (run_id))
         conn.commit()
         cursor.close()
 

tests/test_SubmitPostprocessing.py

@@ -1,6 +1,6 @@
 import requests
 import time
-import psycopg
+import psycopg2
 import pytest
 
 INSTRUMENT = "arcs"
@@ -12,8 +12,8 @@
 class TestPostProcessingAdminView:
     def setup_class(cls):
         # connect to DB
-        cls.conn = psycopg.connect(
-            dbname="workflow",
+        cls.conn = psycopg2.connect(
+            database="workflow",
             user="workflow",
             password="workflow",
             port="5432",

tests/test_livedata.py

@@ -2,7 +2,7 @@
 import os
 import hashlib
 
-import psycopg
+import psycopg2
 import requests
 
 LIVEDATA_TEST_URL = "https://172.16.238.222"
@@ -17,8 +17,8 @@ class TestLiveDataServer:
     @classmethod
     def setup_class(cls):
         """Clean the database before running tests"""
-        conn = psycopg.connect(
-            dbname=os.environ.get("DATABASE_NAME", "workflow"),
+        conn = psycopg2.connect(
+            database=os.environ.get("DATABASE_NAME", "workflow"),
             user=os.environ.get("DATABASE_USER", "workflow"),
             password=os.environ.get("DATABASE_PASS", "workflow"),
             port=os.environ.get("DATABASE_PORT", 5432),

tests/test_setInstrumentPVs.py

@@ -1,5 +1,5 @@
-import psycopg.errors as pge
-import psycopg
+import psycopg2.errors as pge
+import psycopg2
 import argparse
 import os
 import unittest
@@ -14,8 +14,8 @@ def setUp(cls):
 
     @classmethod
     def get_db_conn(cls, options):
-        conn = psycopg.connect(
-            dbname=options.database,
+        conn = psycopg2.connect(
+            database=options.database,
             user=options.user,
             password=options.password,
             host=options.host,
@@ -42,18 +42,17 @@ def insert_test_data(cls):
         cursor = cls.conn.cursor()
         cursor.execute("INSERT INTO report_instrument (name) VALUES (%s);", (instrument,))
         cursor.execute("SELECT id FROM report_instrument where name = %s;", (instrument,))
-        inst_id = cursor.fetchone()[0]
+        inst_id = cursor.fetchone()
         for pv in pvs:
-            cursor.execute("INSERT INTO pvmon_pvname (name, monitored) VALUES (%s, %s);", (pv, "t"))
+            cursor.execute("INSERT INTO pvmon_pvname (name, monitored) VALUES (%s, 't');", (pv,))
         cursor.execute("SELECT id FROM pvmon_pvname where name = ANY(%s);", (pvs,))
         pv_ids = cursor.fetchall()
         for each in pv_ids:
             cursor.execute(
                 "INSERT INTO pvmon_pv (instrument_id, name_id, value, status, update_time) VALUES (%s, %s, 0.0, 0, 0);",
-                (inst_id, each[0]),
+                [inst_id, each],
             )
-            clean_pv_ids.append((inst_id, each[0]))
-
+            clean_pv_ids.append((inst_id[0], each[0]))
         return clean_pv_ids
 
     @classmethod
@@ -66,11 +65,7 @@ def test_monitor_some(self):
         expected_values = [self.ids[0], self.ids[1]]
 
         cursor = self.conn.cursor()
-        cursor.execute(
-            "SELECT setInstrumentPVs('{}'::character varying, ARRAY[{}])".format(
-                test_instrument, ",".join(f"'{pv}'" for pv in test_pvs)
-            )
-        )
+        cursor.execute("SELECT setInstrumentPVs(%s::character varying, ARRAY[%s])", (test_instrument, test_pvs))
         cursor.execute("SELECT instrument_id, pv_name_id FROM pvmon_monitoredvariable;")
         result = cursor.fetchall()
         self.assertListEqual(expected_values, result)
@@ -81,22 +76,19 @@ def test_monitor_all(self):
         expected_values = self.ids
 
         cursor = self.conn.cursor()
-        cursor.execute(
-            "SELECT setInstrumentPVs('{}'::character varying, ARRAY[{}])".format(
-                test_instrument, ",".join(f"'{pv}'" for pv in test_pvs)
-            )
-        )
+        cursor.execute("SELECT setInstrumentPVs(%s::character varying, ARRAY[%s])", (test_instrument, test_pvs))
         cursor.execute("SELECT instrument_id, pv_name_id FROM pvmon_monitoredvariable;")
         result = cursor.fetchall()
         self.assertListEqual(expected_values, result)
 
     def test_monitor_none(self):
         test_instrument = "inst1"
+        test_pvs = []
         expected_values = [(self.ids[0][0], None)]
 
         cursor = self.conn.cursor()
 
-        cursor.execute("SELECT setInstrumentPVs('{}'::character varying, ARRAY[]::text[])".format(test_instrument))
+        cursor.execute("SELECT setInstrumentPVs(%s::character varying, ARRAY[%s])", (test_instrument, test_pvs))
         cursor.execute("SELECT instrument_id, pv_name_id FROM pvmon_monitoredvariable;")
         result = cursor.fetchall()
         self.assertListEqual(expected_values, result)
@@ -108,11 +100,7 @@ def test_monitor_pv_does_not_exist(self):
 
         cursor = self.conn.cursor()
 
-        cursor.execute(
-            "SELECT setInstrumentPVs('{}'::character varying, ARRAY[{}])".format(
-                test_instrument, ",".join(f"'{pv}'" for pv in test_pvs)
-            )
-        )
+        cursor.execute("SELECT setInstrumentPVs(%s::character varying, ARRAY[%s])", (test_instrument, test_pvs))
         cursor.execute("SELECT instrument_id, pv_name_id FROM pvmon_monitoredvariable;")
         result = cursor.fetchall()
         self.assertListEqual(expected_values, result)
@@ -123,8 +111,4 @@ def test_monitor_instrument_does_not_exist(self):
 
         cursor = self.conn.cursor()
         with self.assertRaises(pge.RaiseException):
-            cursor.execute(
-                "SELECT setInstrumentPVs('{}'::character varying, ARRAY[{}])".format(
-                    test_instrument, ",".join(f"'{pv}'" for pv in test_pvs)
-                )
-            )
+            cursor.execute("SELECT setInstrumentPVs(%s::character varying, ARRAY[%s])", (test_instrument, test_pvs))

tests/utils/db.py

@@ -7,7 +7,7 @@ def get_status_queue_id(db_connection, queue_name):
     """
     Return the id for the statusqueue for the provided name
 
-    :param psycopg.Connection db_connection: database connection
+    :param psycopg2.Connection db_connection: database connection
     :param int run_id: ID of the run in table datarun
     :param str queue_name: status queue
     :return: int, ID of the status queue in table statusqueue
@@ -31,14 +31,14 @@ def check_run_status_exist(db_connection, run_id, queue_name):
     """
     Return if the run status was created for the given run_id and queue_name
 
-    :param psycopg.Connection db_connection: database connection
+    :param psycopg2.Connection db_connection: database connection
     :param int run_id: ID of the run in table datarun
     :param str queue_name: status queue
     :return: bool
     """
     cursor = db_connection.cursor()
     queue_id = get_status_queue_id(db_connection, queue_name)
-    cursor.execute("SELECT * FROM report_runstatus WHERE run_id_id = %s AND queue_id_id = %s", (run_id, queue_id))
+    cursor.execute("SELECT * FROM report_runstatus WHERE run_id_id = %s AND queue_id_id = %s", (run_id[0], queue_id))
     result = cursor.fetchone() is not None
     cursor.close()
     return result
@@ -48,22 +48,22 @@ def clear_previous_runstatus(db_connection, run_id):
     """
     Remove all previous run statuses for the given run_id
 
-    :param psycopg.Connection db_connection: database connection
+    :param psycopg2.Connection db_connection: database connection
     :param tuple(int,) run_id: ID of the run in table datarun
     """
     cursor = db_connection.cursor()
     # delete run from tables report_information, report_error and report_runstatus
     cursor.execute(
         "DELETE FROM report_error WHERE run_status_id_id IN (SELECT id FROM report_runstatus "
         "WHERE run_id_id = %s);",
-        (run_id,),
+        run_id,
     )
     cursor.execute(
         "DELETE FROM report_information WHERE run_status_id_id IN (SELECT id FROM report_runstatus "
         "WHERE run_id_id = %s);",
-        (run_id,),
+        run_id,
     )
-    cursor.execute("DELETE FROM report_runstatus WHERE run_id_id = %s;", (run_id,))
+    cursor.execute("DELETE FROM report_runstatus WHERE run_id_id = %s;", run_id)
     db_connection.commit()
     cursor.close()
 
@@ -74,7 +74,7 @@ def add_instrument_data_run(conn, instrument, ipts, run_number, facility="SNS"):
 
     Returns the id for the created rundata
 
-    :param psycopg.Connection conn: database connection
+    :param psycopg2.Connection conn: database connection
     :param str instrument: instrument
     :param str ipts: IPTS identifier, e.g. "IPTS-1234"
     :param int run_number: run number
@@ -128,14 +128,14 @@ def add_instrument_data_run(conn, instrument, ipts, run_number, facility="SNS"):
         conn.commit()
     cursor.close()
 
-    return run_id[0]
+    return run_id
 
 
 def check_error_msg_contains(conn, run_id, error_msg):
     """
     Return if there is an error record for the run that contains error_msg in description
 
-    :param psycopg.Connection conn: database connection
+    :param psycopg2.Connection conn: database connection
     :param int run_id: ID of the run in table datarun
     :param str error_msg: error msg to check for
     :return: bool