WIP: Sanity checks.

micromata · Dec 31, 2024 · addb766 · addb766
1 parent 23bcf7e
commit addb766
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 11 deletions.
diff --git a/projectforge-application/src/main/resources/i18nKeys.json b/projectforge-application/src/main/resources/i18nKeys.json
@@ -2210,8 +2210,8 @@
   {"i18nKey":"system.admin.alertMessage.copyAndPaste.title","bundleName":"I18nResources","translation":"For copy & paste","translationDE":"For copy & paste","usedInClasses":["org.projectforge.web.admin.AdminForm"],"usedInFiles":[]},
   {"i18nKey":"system.admin.button.checkI18nProperties","bundleName":"I18nResources","translation":"Check i18n properties","translationDE":"Check i18n properties","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
   {"i18nKey":"system.admin.button.checkI18nProperties.tooltip","bundleName":"I18nResources","translation":"Check i18n properties for detecting missing translations in different languages.","translationDE":"Check i18n properties for detecting missing translations in different languages.","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
-  {"i18nKey":"system.admin.button.checkJCRSanity","bundleName":"I18nResources","translation":"JCR sanity check","translationDE":"JCR sanity check","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
-  {"i18nKey":"system.admin.button.checkJCRSanity.tooltip","bundleName":"I18nResources","translation":"Checks the whole document repository (JCR) by comparing check sums and file sizes.","translationDE":"Checks the whole document repository (JCR) by comparing check sums and file sizes.","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
+  {"i18nKey":"system.admin.button.checkJCRSanity","bundleName":"I18nResources","translation":"JCR sanity check","translationDE":"JCR sanity check","usedInClasses":[],"usedInFiles":[]},
+  {"i18nKey":"system.admin.button.checkJCRSanity.tooltip","bundleName":"I18nResources","translation":"Checks the whole document repository (JCR) by comparing check sums and file sizes.","translationDE":"Checks the whole document repository (JCR) by comparing check sums and file sizes.","usedInClasses":[],"usedInFiles":[]},
   {"i18nKey":"system.admin.button.checkSystemIntegrity","bundleName":"I18nResources","translation":"Check system integrity","translationDE":"Check system integrity","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
   {"i18nKey":"system.admin.button.checkSystemIntegrity.tooltip","bundleName":"I18nResources","translation":"Some basic checks are done (are there orphaned structure elements in the system?).","translationDE":"Some basic checks are done (are there orphaned structure elements in the system?).","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
   {"i18nKey":"system.admin.button.clearAlertMessage","bundleName":"I18nResources","translation":"Clear alert message","translationDE":"Clear alert message","usedInClasses":[],"usedInFiles":[]},
@@ -2247,7 +2247,6 @@
   {"i18nKey":"system.admin.group.title.misc.logEntries","bundleName":"I18nResources","translation":"Format log entries","translationDE":"Format log entries","usedInClasses":["org.projectforge.web.admin.AdminForm"],"usedInFiles":[]},
   {"i18nKey":"system.admin.group.title.systemChecksAndFunctionality.caches","bundleName":"I18nResources","translation":"Caches","translationDE":"Caches","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
   {"i18nKey":"system.admin.group.title.systemChecksAndFunctionality.configuration","bundleName":"I18nResources","translation":"Configuration","translationDE":"Configuration","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
-  {"i18nKey":"system.admin.group.title.systemChecksAndFunctionality.miscChecks","bundleName":"I18nResources","translation":"Misc checks","translationDE":"Misc checks","usedInClasses":["org.projectforge.web.admin.AdminPage"],"usedInFiles":[]},
   {"i18nKey":"system.admin.logViewer.autoRefresh","bundleName":"I18nResources","translation":"Auto refresh","translationDE":"Auto-Aktualisierung","usedInClasses":["org.projectforge.rest.admin.LogViewFilter"],"usedInFiles":[]},
   {"i18nKey":"system.admin.logViewer.level","bundleName":"I18nResources","translation":"Loglevel","translationDE":"Log-Level","usedInClasses":["org.projectforge.common.logging.LoggingEventData","org.projectforge.rest.admin.LogViewFilter","org.projectforge.rest.admin.LogViewerEvent"],"usedInFiles":[]},
   {"i18nKey":"system.admin.logViewer.loggerName","bundleName":"I18nResources","translation":"Logger name","translationDE":"Quelle","usedInClasses":["org.projectforge.common.logging.LoggingEventData"],"usedInFiles":[]},

diff --git a/.../src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotDO.kt b/.../src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotDO.kt
@@ -23,12 +23,14 @@
 
 package org.projectforge.business.fibu.orderbooksnapshots
 
+import com.fasterxml.jackson.annotation.JsonIgnore
 import jakarta.persistence.*
+import org.projectforge.framework.json.JsonUtils
 import java.time.LocalDate
-import java.util.Date
+import java.util.*
 
 /**
- * SELECT date, created, incremental_based_on, octet_length(serialized_orderbook) AS byte_count FROM t_fibu_orderbook_snapshots;
+ * SELECT date, created, incremental_based_on, octet_length(serialized_orderbook) AS byte_count, size FROM t_fibu_orderbook_snapshots;
  * @author Kai Reinhard ([email protected])
  */
 @Entity
@@ -37,9 +39,18 @@ import java.util.Date
     uniqueConstraints = [UniqueConstraint(columnNames = ["date"])],
 )
 @NamedQueries(
-    NamedQuery(name = OrderbookSnapshotDO.FIND_META_BY_DATE, query = "select date as date,incrementalBasedOn as incrementalBasedOn from OrderbookSnapshotDO where date=:date"),
-    NamedQuery(name = OrderbookSnapshotDO.SELECT_ALL_METAS, query = "select date as date,incrementalBasedOn as incrementalBasedOn from OrderbookSnapshotDO order by date desc"),
-    NamedQuery(name = OrderbookSnapshotDO.SELECT_ALL_FULLBACKUP_METAS, query = "select date as date,incrementalBasedOn as incrementalBasedOn from OrderbookSnapshotDO where incrementalBasedOn is null order by date desc"),
+    NamedQuery(
+        name = OrderbookSnapshotDO.FIND_META_BY_DATE,
+        query = "select date as date,incrementalBasedOn as incrementalBasedOn,size as size from OrderbookSnapshotDO where date=:date"
+    ),
+    NamedQuery(
+        name = OrderbookSnapshotDO.SELECT_ALL_METAS,
+        query = "select date as date,incrementalBasedOn as incrementalBasedOn,size as size from OrderbookSnapshotDO order by date desc"
+    ),
+    NamedQuery(
+        name = OrderbookSnapshotDO.SELECT_ALL_FULLBACKUP_METAS,
+        query = "select date as date,incrementalBasedOn as incrementalBasedOn,size as size from OrderbookSnapshotDO where incrementalBasedOn is null order by date desc"
+    ),
 )
 internal class OrderbookSnapshotDO {
     @get:Id
@@ -55,6 +66,7 @@ internal class OrderbookSnapshotDO {
      */
     @get:Column(name = "serialized_orderbook", columnDefinition = "BLOB")
     @get:Basic(fetch = FetchType.LAZY) // Lazy isn't reliable for byte arrays.
+    @JsonIgnore
     var serializedOrderBook: ByteArray? = null
 
     /**
@@ -63,10 +75,17 @@ internal class OrderbookSnapshotDO {
     @get:Column(name = "incremental_based_on")
     var incrementalBasedOn: LocalDate? = null
 
+    @get:Column
+    var size: Int? = null
+
     @get:Transient
     val incremental: Boolean
         get() = incrementalBasedOn != null
 
+    override fun toString(): String {
+        return JsonUtils.toJson(this)
+    }
+
     companion object {
         internal const val FIND_META_BY_DATE = "OrderSnapshotsDO_FindMetaByDate"
         internal const val SELECT_ALL_METAS = "OrderSnapshotsDO_SelectAllMetas"

diff --git a/...kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsSanityCheck.kt b/...kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsSanityCheck.kt
@@ -23,11 +23,32 @@
 
 package org.projectforge.business.fibu.orderbooksnapshots
 
+import org.projectforge.common.extensions.format
+import org.projectforge.common.extensions.formatBytes
 import org.projectforge.jobs.AbstractJob
 import org.projectforge.jobs.JobExecutionContext
 
 class OrderbookSnapshotsSanityCheck(val orderbookSnapshotsService: OrderbookSnapshotsService) :
     AbstractJob("Checks the recent order book' snapshots.") {
     override fun execute(jobContext: JobExecutionContext) {
+        val entries = orderbookSnapshotsService.selectMetas()
+        val fullSnapshots = entries.count { it.incrementalBasedOn == null }
+        val incrementalSnapshots = entries.count { it.incrementalBasedOn != null }
+        val totalSize = entries.sumOf { it.size ?: 0 }
+        jobContext.addMessage("Found ${entries.size} order book snapshots: total-size=${totalSize.formatBytes()}, full=${fullSnapshots.format()}, incremental=${incrementalSnapshots.format()}.")
+        // Test all last 10 snapshots:
+        entries.take(10).forEach {
+            val date = it.date
+            if (date == null) {
+                jobContext.addError("Date is null for entry: ${it}")
+                return@forEach
+            }
+            try {
+                orderbookSnapshotsService.readSnapshot(date)
+                jobContext.addMessage("Snapshot for date $date (${it.size.formatBytes()}) is readable.")
+            } catch (e: Exception) {
+                jobContext.addError("Error reading snapshot for date $date: $e")
+            }
+        }
     }
 }
diff --git a/...ain/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsService.kt b/...ain/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsService.kt
@@ -171,6 +171,7 @@ class OrderbookSnapshotsService {
         OrderbookSnapshotDO().also {
             it.date = date
             it.serializedOrderBook = rawSnapshot.gzBytes
+            it.size = it.serializedOrderBook?.size
             it.incrementalBasedOn = incrementalBasedOn
         }.let {
             persistenceService.runInTransaction { context ->
@@ -239,11 +240,12 @@ class OrderbookSnapshotsService {
             OrderbookSnapshotDO().also { result ->
                 result.date = TupleUtils.getLocalDate(it, "date")
                 result.incrementalBasedOn = TupleUtils.getLocalDate(it, "incrementalBasedOn")
+                result.size = TupleUtils.getInt(it, "size")
             }
         }
     }
 
-    private fun selectMetas(onlyFullBackups: Boolean = false): List<OrderbookSnapshotDO> {
+    internal fun selectMetas(onlyFullBackups: Boolean = false): List<OrderbookSnapshotDO> {
         val named =
             if (onlyFullBackups) OrderbookSnapshotDO.SELECT_ALL_FULLBACKUP_METAS else OrderbookSnapshotDO.SELECT_ALL_METAS
         val res = persistenceService.executeNamedQuery(
@@ -253,6 +255,7 @@ class OrderbookSnapshotsService {
             OrderbookSnapshotDO().also { result ->
                 result.date = TupleUtils.getLocalDate(it, "date")
                 result.incrementalBasedOn = TupleUtils.getLocalDate(it, "incrementalBasedOn")
+                result.size = TupleUtils.getInt(it, "size")
             }
         }
         return if (onlyFullBackups) {

diff --git a/...-business/src/main/resources/flyway/migrate/hsqldb/V8.0.5__RELEASE-OrderbookSnapshots.sql b/...-business/src/main/resources/flyway/migrate/hsqldb/V8.0.5__RELEASE-OrderbookSnapshots.sql
@@ -3,7 +3,8 @@ CREATE TABLE t_fibu_orderbook_snapshots
     date                 DATE NOT NULL,
     created              TIMESTAMP WITHOUT TIME ZONE NOT NULL,
     incremental_based_on DATE,
-    serialized_orderbook BLOB
+    serialized_orderbook BLOB,
+    size                 INTEGER
 );
 
 ALTER TABLE t_fibu_orderbook_snapshots

diff --git a/...-business/src/main/resources/flyway/migrate/postgresql/V8.0.5__RELEASE-OrderSnapshots.sql b/...-business/src/main/resources/flyway/migrate/postgresql/V8.0.5__RELEASE-OrderSnapshots.sql
@@ -3,7 +3,8 @@ CREATE TABLE t_fibu_orderbook_snapshots
     date                 DATE NOT NULL,
     created              TIMESTAMP WITHOUT TIME ZONE NOT NULL,
     incremental_based_on DATE,
-    serialized_orderbook BYTEA
+    serialized_orderbook BYTEA,
+    size                 INTEGER
 );
 
 ALTER TABLE t_fibu_orderbook_snapshots