feat: upload compressed sqlite dump to s3

RHINENG-13599

pyproject.toml

@@ -48,6 +48,7 @@ schema = ">=0.7.3"
 uvicorn = "^0.25.0"
 watchtower = ">=1.0.0"
 attrs = "^23.2.0"
+zstandard = "^0.23.0"
 
 [tool.poetry.group.dev.dependencies]
 coverage = ">=4.5.3"

vmaas/common/config.py

@@ -72,6 +72,17 @@ def clowder(self) -> None:
         self.cw_aws_region = self._cfg.logging.cloudwatch.region
         self.cw_aws_log_group = self._cfg.logging.cloudwatch.logGroup
 
+        s3_buckets = getattr(self._cfg.objectStore, "buckets", [])
+        self.s3_available = bool(s3_buckets)
+        self.dump_bucket_name = s3_buckets[0].requestedName if self.s3_available else None
+        self.dump_s3_tls = getattr(self._cfg.objectStore, "tls", "")
+        s3_scheme = "http"
+        if self.dump_s3_tls:
+            s3_scheme += "s"
+        self.dump_s3_url = f"{s3_scheme}://{getattr(self._cfg.objectStore, 'hostname', '')}:{getattr(self._cfg.objectStore, 'port', '')}"
+        self.dump_s3_access_key = getattr(self._cfg.objectStore, "accessKey", "")
+        self.dump_s3_secret_key = getattr(self._cfg.objectStore, "secretKey", "")
+
         self.tls_ca_path = self._cfg.tlsCAPath
         self.reposcan_url = ""
         self.webapp_go_url = ""

vmaas/reposcan/exporter.py

@@ -9,14 +9,26 @@
 import os
 import sqlite3
 
+import zstandard as zstd
+
+from boto3 import client as boto3_client
+from botocore.exceptions import ClientError
+
+from vmaas.common.config import Config
 from vmaas.common.logging_utils import get_logger, init_logging
 from vmaas.common.date_utils import format_datetime, now
 from vmaas.common.fileutil import remove_file_if_exists
 from vmaas.reposcan.database.database_handler import DatabaseHandler, NamedCursor, init_db
 
 DEFAULT_KEEP_COPIES = "2"
 DUMP = '/data/vmaas.db'
+DUMP_COMPRESSED = f"{DUMP}.zstd"
 LOGGER = get_logger(__name__)
+CFG = Config()
+
+
+class S3UploadError(Exception):
+    """S3 Error exception."""
 
 
 def fetch_latest_dump():
@@ -27,6 +39,36 @@ def fetch_latest_dump():
         return None
 
 
+def upload_dump_s3():
+    """Upload DB dump to S3 bucket."""
+    try:
+        client = boto3_client(
+            "s3",
+            endpoint_url=CFG.dump_s3_url,
+            aws_access_key_id=CFG.dump_s3_access_key,
+            aws_secret_access_key=CFG.dump_s3_secret_key,
+            use_ssl=CFG.dump_s3_tls,
+        )
+    except ClientError as err:
+        raise S3UploadError("Failed to create boto3 client") from err
+
+    try:
+        cctx = zstd.ZstdCompressor()
+        with open(DUMP, "rb") as ifd, open(DUMP_COMPRESSED, "wb") as ofd:
+            cctx.copy_stream(ifd, ofd)
+    except (ValueError, MemoryError, zstd.ZstdError) as err:
+        raise S3UploadError("Failed to compress vmaas.db") from err
+
+    try:
+        client.upload_file(
+            Filename=DUMP_COMPRESSED,
+            Bucket=CFG.dump_bucket_name,
+            Key="vmaas.db.zstd",
+        )
+    except ClientError as err:
+        raise S3UploadError("Failed to upload vmaas.db to s3") from err
+
+
 class SqliteDump:
     """Class for creating sqlite disk dump from database."""