Merge branch 'main' into line-numbers

.github/workflows/build-and-push.yaml

@@ -11,6 +11,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: git fetch
+        run: |
+          git fetch
+      - name: git checkout
+        run: |
+          git checkout ${{ github.head_ref }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 

.gitignore

@@ -19,3 +19,4 @@ config.yaml
 terraform.tfstate*
 .terraform*
 tofu/kube.config
+ansible/collections/*

ansible/ansible.cfg

@@ -1,5 +1,13 @@
 [defaults]
-
 # Better error output
 stdout_callback=debug
 stderr_callback=debug
+
+collections_path=./collections/ansible_collections
+
+# we're only using localhost, no need for the warning.
+localhost_warning=False
+
+[inventory]
+# Only using localhost, so no inventory
+inventory_unparsed_warning=False

deploy.sh

@@ -22,13 +22,18 @@ source secrets.sh
 python3 -m venv .venv/deploy
 source .venv/deploy/bin/activate
 pip install ansible==8.1.0 kubernetes==26.1.0
+# install helm diff. Needed to keep helm module idempotent
+helm plugin install https://github.com/databus23/helm-diff || true
 
 cd tofu
 AWS_ACCESS_KEY_ID=${ACCESS_KEY} AWS_SECRET_ACCESS_KEY=${SECRET_KEY} tofu init
 AWS_ACCESS_KEY_ID=${ACCESS_KEY} AWS_SECRET_ACCESS_KEY=${SECRET_KEY} tofu apply # -auto-approve
 export KUBECONFIG=$(pwd)/kube.config
 
 cd ../ansible
+# install collections here to take advantage of ansible.cfg configs
+ansible-galaxy collection install -U kubernetes.core -p ./collections
+
 ansible-playbook quarry.yaml
 #kubectl create namespace quarry --dry-run=client -o yaml | kubectl apply -f -
 #helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml

quarry/default_config.yaml

@@ -40,6 +40,11 @@ TOOLS_DB_PORT: 3306
 TOOLS_DB_USER: ''
 TOOLS_DB_PASSWORD: ''
 
+QUARRY_P_HOST: 'db'
+QUARRY_P_PORT: 3306
+QUARRY_P_USER: ''
+QUARRY_P_PASSWORD: ''
+
 OUTPUT_PATH_TEMPLATE: '/results/%s/%s/%s.sqlite'
 REDIS_HOST: 'redis'
 REDIS_PORT: 6379

quarry/web/replica.py

@@ -13,13 +13,17 @@ def __init__(self, config):
 
     def _db_name_mangler(self):
         self.is_tools_db = False
+        self.is_quarry_p = False
         if self.dbname == "":
             raise ReplicaConnectionException(
                 "Attempting connection before a database is selected"
             )
         if "__" in self.dbname and self.dbname.endswith("_p"):
             self.is_tools_db = True
             self.database_p = self.dbname
+        elif self.dbname == "quarry" or self.dbname == "quarry_p":
+            self.is_quarry_p = True
+            self.database_p = "quarry_p"
         elif self.dbname == "meta" or self.dbname == "meta_p":
             self.database_name = "s7"
             self.database_p = "meta_p"
@@ -41,6 +45,8 @@ def _db_name_mangler(self):
     def get_host_name(self):
         if self.is_tools_db:
             return self.config["TOOLS_DB_HOST"]
+        if self.is_quarry_p:
+            return self.config["DB_HOST"]
         if self.config["REPLICA_DOMAIN"]:
             return f"{self.database_name}.{self.config['REPLICA_DOMAIN']}"
         return self.database_name
@@ -62,7 +68,12 @@ def connection(self, db):
         self.dbname = db
         self._db_name_mangler()
         host = self.get_host_name()
-        conf_prefix = "TOOLS_DB" if self.is_tools_db else "REPLICA"
+        if self.is_tools_db:
+            conf_prefix = "TOOLS_DB"
+        elif self.is_quarry_p:
+            conf_prefix = "QUARRY_P"
+        else:
+            conf_prefix = "REPLICA"
         port = self.config[f"{conf_prefix}_PORT"]
         connect_opts = {
             "db": self.database_p,

quarry/web/utils/__init__.py

@@ -2,7 +2,7 @@
 
 
 VALID_DB_NAMES = re.compile(
-    r"^(?:(?:(?:centralauth|meta|[0-9a-z_]*wik[a-z]+)(?:_p)?)|quarry|s\d+__\w+_p)$"
+    r"^(?:(?:(?:centralauth|meta|[0-9a-z_]*wik[a-z]+)(?:_p)?)|quarry(?:_p)?|s\d+__\w+_p)$"
 )
 
 

quarry/web/worker.py

@@ -27,8 +27,13 @@
 
 
 def get_replag(cur):
-    cur.execute("SELECT lag FROM heartbeat_p.heartbeat;")
-    return int(cur.fetchall()[0][0])
+    cur.execute("SELECT * FROM information_schema.tables WHERE table_schema='heartbeat_p' and table_name='heartbeat';")
+    if cur.rowcount:
+        cur.execute("SELECT lag FROM heartbeat_p.heartbeat;")
+        return int(cur.fetchall()[0][0])
+    else:
+        # there is not a heartbeat table on this database
+        return 0
 
 
 @worker_process_init.connect

schema.sql

@@ -59,3 +59,9 @@ CREATE TABLE IF NOT EXISTS star(
 CREATE INDEX IF NOT EXISTS star_user_id_index ON star(user_id);
 CREATE INDEX IF NOT EXISTS star_query_id_index ON star(query_id);
 CREATE UNIQUE INDEX IF NOT EXISTS star_user_query_index ON star(user_id, query_id);
+
+CREATE DATABASE IF NOT EXISTS quarry_p CHARACTER SET utf8;
+CREATE VIEW IF NOT EXISTS quarry_p.query AS SELECT * FROM quarry.query;
+CREATE VIEW IF NOT EXISTS quarry_p.query_revision AS SELECT * FROM quarry.query_revision;
+CREATE VIEW IF NOT EXISTS quarry_p.query_run AS SELECT * FROM quarry.query_run;
+CREATE VIEW IF NOT EXISTS quarry_p.star AS SELECT * FROM quarry.star;

tofu/127a.tf

@@ -0,0 +1,40 @@
+resource "openstack_containerinfra_cluster_v1" "k8s_127a" {
+  name                = "quarry-127a"
+  cluster_template_id = resource.openstack_containerinfra_clustertemplate_v1.template_127a.id
+  master_count        = 1
+  node_count          = 2
+}
+
+resource "local_file" "kube_config" {
+  content  = resource.openstack_containerinfra_cluster_v1.k8s_127a.kubeconfig.raw_config
+  filename = "kube.config"
+}
+
+resource "openstack_containerinfra_clustertemplate_v1" "template_127a" {
+  name                  = "quarry-127a"
+  coe                   = "kubernetes"
+  dns_nameserver        = "8.8.8.8"
+  docker_storage_driver = "overlay2"
+  docker_volume_size    = 20
+  external_network_id   = "wan-transport-eqiad"
+  fixed_subnet          = "cloud-instances2-b-eqiad"
+  fixed_network         = "lan-flat-cloudinstances2b"
+  flavor                = "g4.cores4.ram8.disk20"
+  floating_ip_enabled   = "false"
+  image                 = "Fedora-CoreOS-38"
+  master_flavor         = "g4.cores2.ram4.disk20"
+  network_driver        = "calico"
+
+  labels = {
+    kube_tag                       = "v1.27.8-rancher2"
+    container_runtime              = "containerd"
+    containerd_version             = "1.6.28"
+    containerd_tarball_sha256      = "f70736e52d61e5ad225f4fd21643b5ca1220013ab8b6c380434caeefb572da9b"
+    cloud_provider_tag             = "v1.27.3"
+    cinder_csi_plugin_tag          = "v1.27.3"
+    k8s_keystone_auth_tag          = "v1.27.3"
+    magnum_auto_healer_tag         = "v1.27.3"
+    octavia_ingress_controller_tag = "v1.27.3"
+    calico_tag                     = "v3.26.4"
+  }
+}

tofu/vars.tf

@@ -9,5 +9,5 @@ variable "tenant_id" {
 }
 variable "application_credential_id" {
   type = string
-  default = "4917ce71b98e498e8a6c5814b095b28e"
+  default = "446ab9c3713b4a0b8c2da540021dd314"
 }