MCOL-6032: check that required directories are in the correct state

.drone.jsonnet

@@ -493,6 +493,31 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     ],
   },
 
+  multinode_mtrlog:: {
+    name: "multinode-mtrlog",
+    depends_on: ["mtr"],
+    image: "docker:28.2.2",
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
+    when: {
+      status: ["success", "failure"],
+    },
+    commands: [
+      "apk add bash",
+      "mkdir -p /drone/src/" + result + "/mtr-multinode/mcs1 /drone/src/" + result + "/mtr-multinode/mcs2 /drone/src/" + result + "/mtr-multinode/mcs3",
+      "docker cp mcs1:/var/log/mariadb/columnstore/cmapi_server.log /drone/src/" + result + "/mtr-multinode/mcs1/ 2>/dev/null || true",
+      "docker cp mcs1:/var/log/mariadb/columnstore/debug.log /drone/src/" + result + "/mtr-multinode/mcs1/ 2>/dev/null || true",
+      "docker exec -t mcs1 journalctl -u mariadb --no-pager > /drone/src/" + result + "/mtr-multinode/mcs1/journalctl_mariadb.log 2>&1 || true",
+      "docker cp mcs2:/var/log/mariadb/columnstore/cmapi_server.log /drone/src/" + result + "/mtr-multinode/mcs2/ 2>/dev/null || true",
+      "docker cp mcs2:/var/log/mariadb/columnstore/debug.log /drone/src/" + result + "/mtr-multinode/mcs2/ 2>/dev/null || true",
+      "docker exec -t mcs2 journalctl -u mariadb --no-pager > /drone/src/" + result + "/mtr-multinode/mcs2/journalctl_mariadb.log 2>&1 || true",
+      "docker cp mcs3:/var/log/mariadb/columnstore/cmapi_server.log /drone/src/" + result + "/mtr-multinode/mcs3/ 2>/dev/null || true",
+      "docker cp mcs3:/var/log/mariadb/columnstore/debug.log /drone/src/" + result + "/mtr-multinode/mcs3/ 2>/dev/null || true",
+      "docker exec -t mcs3 journalctl -u mariadb --no-pager > /drone/src/" + result + "/mtr-multinode/mcs3/journalctl_mariadb.log 2>&1 || true",
+      "docker cp mcs1:/usr/share/mysql-test/var/log /drone/src/" + result + "/mtr-multinode/mcs1/mtr-logs 2>/dev/null || true",
+      "ls -lR /drone/src/" + result + "/mtr-multinode",
+    ],
+  },
+
   kind: "pipeline",
   type: "docker",
   name: std.join(" ", [branch, platform, event, arch, server, customBootstrapParamsKey, customBuildEnvCommandsMapKey]),
@@ -632,7 +657,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
          [pipeline.cmapitest] +
          [pipeline.cmapilog] +
          [pipeline.publish("cmapilog")] +
-         (if (platform == "rockylinux:8" && arch == "amd64" && customBootstrapParamsKey == "gcc-toolset") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) +
+         (if (platform == "rockylinux:8" && arch == "amd64" && customBootstrapParamsKey == "gcc-toolset") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] + [pipeline.multinode_mtrlog] + [pipeline.publish("multinode-mtrlog")] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) +
          [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] +
          [pipeline.regressionlog] +
          [pipeline.publish("regressionlog")] +

cmapi/.gitignore

@@ -89,8 +89,9 @@ ubuntu20.04
 buildinfo.txt
 
 # Self-signed certificates
-cmapi_server/self-signed.crt
-cmapi_server/self-signed.key
+self-signed.crt
+self-signed.key
+
 # Comes in handy if you build packages locally
 pp
 mcs

cmapi/cmapi_server/__main__.py

@@ -29,6 +29,7 @@
 from cmapi_server.managers.application import AppManager
 from cmapi_server.managers.process import MCSProcessManager
 from cmapi_server.managers.certificate import CertificateManager
+from cmapi_server.invariant_checks import run_invariant_checks
 from failover.node_monitor import NodeMonitor
 from mcs_node_control.models.dbrm_socket import SOCK_TIMEOUT, DBRMSocketHandler
 from mcs_node_control.models.node_config import NodeConfig
@@ -151,6 +152,12 @@ def stop(self):
     CertificateManager.create_self_signed_certificate_if_not_exist()
     CertificateManager.renew_certificate()
 
+    # Run checks, if some of them fail -- log and exit
+    diag = run_invariant_checks()
+    if diag:
+        logging.error('Invariant checks failed, exiting')
+        sys.exit(1)
+
     app = cherrypy.tree.mount(root=None, config=CMAPI_CONF_PATH)
     root_config = {
         "request.dispatch": dispatcher,

cmapi/cmapi_server/controllers/endpoints.py

@@ -44,6 +44,7 @@
 from cmapi_server.managers.process import MCSProcessManager, MDBProcessManager
 from cmapi_server.managers.transaction import TransactionManager
 from cmapi_server.node_manipulation import is_master, switch_node_maintenance
+from cmapi_server.invariant_checks import run_invariant_checks
 
 # Bug in pylint https://github.com/PyCQA/pylint/issues/4584
 requests.packages.urllib3.disable_warnings()  # pylint: disable=no-member
@@ -447,6 +448,15 @@ def put_config(self):
                 sm_config_filename=sm_config_filename,
                 sm_config_string=sm_config
             )
+
+            diag = run_invariant_checks()
+            if diag:
+                raise_422_error(
+                    module_logger, func_name,
+                    f'Invariant checks failed. Details:\n{diag.strip()}',
+                    exc_info=False
+                )
+
             # TODO: change stop/start to restart option.
             try:
                 MCSProcessManager.stop_node(

cmapi/cmapi_server/invariant_checks.py

@@ -0,0 +1,97 @@
+import logging
+import os
+from typing import Optional, Tuple
+
+from mr_kot import Runner, Status, any_of, check, check_all, fact, parametrize
+from mr_kot_fs_validators import Exists, GroupIs, HasMode, IsDir, OwnerIs
+
+from cmapi_server import helpers
+from cmapi_server.constants import MCS_DATA_PATH
+from mcs_node_control.models.node_config import NodeConfig
+
+logger = logging.getLogger(__name__)
+
+
+def run_invariant_checks() -> Optional[str]:
+    """Run invariant checks, log results, and return a formatted string with problems, if any.
+    """
+    logger.info('Starting invariant checks')
+    runner = Runner()
+    result = runner.run()
+    problems = result.problems()
+
+    # Log each fail/error/warning for diagnostics
+    diag = ''
+    for problem in problems:
+        fn = logger.warning if problem.status == Status.WARN else logger.error
+        fn(
+            'Invariant check with id=%s produced %s: %r',
+            problem.id, problem.status, problem.evidence,
+        )
+        diag += f'{problem.id}: {problem.evidence}\n'
+
+    logger.info(
+        'Stats: overall=%s counts=%s',
+        result.overall.value,
+        {k.value: v for k, v in result.counts.items() if v != 0}
+    )
+    if result.overall in (Status.FAIL, Status.ERROR):
+        logger.error('Invariant checks failed')
+        return diag
+    else:
+        logger.info('Invariant checks passed')
+
+
+### Facts
+@fact
+def storage_type() -> str:
+    """Provides storage type: shared_fs or s3."""
+    return 's3' if NodeConfig().s3_enabled() else 'shared_fs'
+
+@fact
+def is_shared_fs(storage_type: str) -> bool:
+    return storage_type == 'shared_fs'
+
+@fact
+def dispatcher_name() -> str:
+    """Provides environment dispatcher name: systemd or container"""
+    cfg = helpers.get_config_parser()
+    name, _ = helpers.get_dispatcher_name_and_path(cfg)
+    return name
+
+@fact
+def is_systemd_disp(dispatcher_name: str) -> bool:
+    return dispatcher_name == 'systemd'
+
+
+### Checks
+REQUIRED_LOCAL_DIRS = [
+    os.path.join(MCS_DATA_PATH, 'data1'),
+    os.path.join(MCS_DATA_PATH, 'data1', 'systemFiles'),
+    os.path.join(MCS_DATA_PATH, 'data1', 'systemFiles', 'dbrm'),
+]
+
+@check(selector='is_shared_fs')
+@parametrize('dir', values=REQUIRED_LOCAL_DIRS, fail_fast=True)
+def required_dirs_perms(dir: str) -> Tuple[Status, str]:
+    status, ev = check_all(
+        dir,
+        Exists(),
+        IsDir(),
+        HasMode('1755'),
+    )
+    return (status, ev)
+
+@check(selector='is_shared_fs, is_systemd_disp')
+@parametrize('dir', values=REQUIRED_LOCAL_DIRS, fail_fast=True)
+def required_dirs_ownership(dir: str) -> Tuple[Status, str]:
+    # Check ownership only when not in containers
+    status, ev = check_all(
+        dir,
+        Exists(),
+        # The correct owner is mysql, but i've seen mariadb as owner of the mountpoint,
+        #   so we allow both
+        any_of(OwnerIs('mysql'), OwnerIs('mariadb')),
+        any_of(GroupIs('mysql'), GroupIs('mariadb')),
+    )
+    return (status, ev)

cmapi/pyproject.toml

@@ -9,6 +9,7 @@ select = [
     "B",  # flake8-bugbear: common bugs and anti-patterns
     "UP", # pyupgrade: use modern Python syntax
     "N",  # pep8-naming: naming conventions
+    "Q",  # flake8-quotes: enforce quote style
 ]
 
 ignore = []
@@ -22,5 +23,11 @@ exclude = [
 quote-style = "single"
 
 [tool.ruff.lint.isort]
+known-first-party = ["cmapi_server", "failover", "mcs_node_control", "tracing"]
 force-single-line = false
-combine-as-imports = true
+combine-as-imports = true
+
+[tool.ruff.lint.flake8-quotes]
+inline-quotes = "single"
+multiline-quotes = "single"
+docstring-quotes = "double"

cmapi/requirements.in

@@ -13,8 +13,11 @@ psutil==7.0.0
 pyotp==2.9.0
 requests==2.32.3
 # required for CherryPy RoutesDispatcher,
-# but CherryPy itself has no such a dependency
+# but CherryPy itself has no such dependency
 Routes==2.5.1
 typer==0.15.2
 pydantic==2.11.7
 sentry-sdk==2.34.1
+# Invariant checks
+mr_kot==0.9.2
+mr_kot_fs_validators==0.2.0

cmapi/requirements.txt

@@ -144,6 +144,10 @@ more-itertools==10.7.0
     #   cherrypy
     #   jaraco-functools
     #   jaraco-text
+mr-kot==0.9.2
+    # via -r requirements.in
+mr-kot-fs-validators==0.2.0
+    # via -r requirements.in
 multidict==6.6.4
     # via
     #   aiohttp