Merge pull request #253 from micromata/Release-8.1-SNAPSHOT

Release 8.1 snapshot WIP: Order book snapshots, ForecastExport fixed (customer and project name were missing)
micromata · Jan 3, 2025 · f6e0ad5 · f6e0ad5
2 parents 656529e + 7163b8c
commit f6e0ad5
Show file tree

Hide file tree

Showing 13 changed files with 169 additions and 68 deletions.
diff --git a/README.adoc b/README.adoc
@@ -96,7 +96,7 @@ spring.datasource.driver-class-name=org.postgresql.Driver
 |===
 |Stop|`docker stop projectforge-postgres`
 |Start|`docker start projectforge-postgres`
-|Import dump (optional)|`docker run -v ~/ProjectForgeBackup/pf.sql:/mnt/pf.sql -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 psql -h postgres -U projectforge -q -f /mnt/pf.sql  2>&1 > log.txt`
+|Import dump (optional)|`docker run -v ~/ProjectForgeBackup/pf.sql:/mnt/pf.sql -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 psql -h postgres -U projectforge -q -f /mnt/pf.sql`
 |PSQL|`docker run -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 psql -h postgres -U projectforge`
 |Reset passwords (optional)|`update t_pf_user_password SET password_hash='SHA{BC871652288E56E306CFA093BEFC3FFCD0ED8872}', password_salt=null; update t_pf_user SET password='SHA{BC871652288E56E306CFA093BEFC3FFCD0ED8872}', password_salt=null, email='m.developer@localhost';` +
 password is now `test123`.

diff --git a/ToDo.adoc b/ToDo.adoc
@@ -1,5 +1,6 @@
 Aktuell:
 - Scripting: Ergebnis Unresolved reference 'memo', 'todo'.: line 94 to 94 (add only activated plugins)
+- Summen in DB-Exporten eintragen und serverseitig evaluieren.
 - Groovy-scripts: remove or fix.
 - AG-Grid: setColumnStates wird nicht in den UserPrefs gespeichert.
 - Wicket: Auftragsbuch: org.apache.wicket.core.request.mapper.StalePageException: A request to page '[Page class = org.projectforge.web.fibu.AuftragEditPage, id = 9, render count = 3]' has been made with stale 'renderCount'. The page will be re-rendered.
@@ -25,10 +26,7 @@ Aktuell:
 org.hibernate.persister.entity.AbstractEntityPersister#generateSelectLazy:
 
 History
-- Positionen etc. vernünftig anzeigen.
-
 - Suche:
-  - Suche nach K+S (aktuell noch Provisorium)
 - HistoryConvertContext: Am Ende alle displayProperties übernehmen.
 
 - ProjektEdit: History enthält keine neuen Kost2DOs/Arten (Einträge sind aber da).
@@ -43,22 +41,13 @@ History
 - History of AddressCampaignValueDO's (AddressCampaignValueDao.convertToDisplayHistoryEntries removed)
 
 Später
-- Hibernate-Search: K+S etc.
-- Besuchsbuch und EmployeeValidSinceAttr: DisplayEntries von Besuchtagen mit Datum des Besuchs versehen.
+- Fakturaquote
 - Suche-Seite hat veraltete Bereiche, AddressListPage läuft auf Fehler.
 - OrderExport: paymentSchedules werden gefetcht.
-- Update caches after single load or modification.
 
 Ganz später
 - Kalenderlist ruft x-fach DB: FIND GroupDO resultClass=GroupDO auf.
 
-Postgresql-Dump-Imports bechleunigen:
-
-ALTER SYSTEM SET fsync = off;
-ALTER SYSTEM SET synchronous_commit = off;
-SET maintenance_work_mem = '512MB';
-drop view v_t_pf_user;
-
 Rancher
 
 docker system df
@@ -67,13 +56,23 @@ docker system df
 docker volume ls
 docker volume rm <volume-name>
 
-drop table t_fibu_orderbook_storage;
-delete from t_flyway_schema_history where version = '2023.11.01';
+
+Postgresql-Dump-Imports bechleunigen:
+
+ALTER SYSTEM SET fsync = off;
+ALTER SYSTEM SET synchronous_commit = off;
+SET maintenance_work_mem = '512MB';
+
+
+drop view v_t_pf_user;
+
 
 Orderbooks importieren:
 docker cp ~/ProjectForgeBackup/ProjectForge-Orderbook_*.gz projectforge-postgres:/tmp/
 
 \set file_path '/tmp/ProjectForge-Orderbook_2023-11-01.gz'
-INSERT INTO t_fibu_orderbook_storage (date, serialized_orderbook)  VALUES ('2023-11-01', pg_read_binary_file(:'file_path')::bytea);
-\set file_path '/tmp/ProjectForge-Orderbook_2023-12-01.gz'
-INSERT INTO t_fibu_orderbook_storage (date, serialized_orderbook)  VALUES ('2023-12-01', pg_read_binary_file(:'file_path')::bytea);
+INSERT INTO t_fibu_orderbook_snapshots (date, created, serialized_orderbook, size)  VALUES ('2023-11-01', NOW(), pg_read_binary_file(:'file_path')::bytea, (pg_stat_file(:'file_path')).size);
+
+docker run -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 pg_dump -h postgres -U projectforge --data-only --column-inserts  --table=t_fibu_orderbook_snapshots
+
+psql -f export.sql
diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/AuftragsCacheService.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/AuftragsCacheService.kt
@@ -66,8 +66,15 @@ class AuftragsCacheService {
                     order.bemerkung = getString(tuple, "bemerkung")
                     order.periodOfPerformanceBegin = getLocalDate(tuple, "periodOfPerformanceBegin")
                     order.periodOfPerformanceEnd = getLocalDate(tuple, "periodOfPerformanceEnd")
+                    order.kundeText = getString(tuple, "kundeText")
+                    getLong(tuple, "kundeId")?.let { kundeId ->
+                        order.kunde = em.getReference(KundeDO::class.java, kundeId)
+                    }
+                    getLong(tuple, "projektId")?.let { projektId ->
+                        order.projekt = em.getReference(ProjektDO::class.java, projektId)
+                    }
                     getLong(tuple, "contactPersonId")?.let { userId ->
-                        order.contactPerson = PFUserDO().also { it.id = userId }
+                        order.contactPerson = em.getReference(PFUserDO::class.java, userId)
                     }
                 }
             }
@@ -82,7 +89,7 @@ class AuftragsCacheService {
                 AuftragsPositionDO().also { pos ->
                     pos.id = getLong(tuple, "id")
                     getLong(tuple, "auftragId")?.let { auftragId ->
-                        pos.auftrag = AuftragDO().also { it.id = auftragId }
+                        pos.auftrag = em.getReference(AuftragDO::class.java, auftragId)
                     }
                     pos.number = getShort(tuple, "number")!!
                     pos.deleted = getBoolean(tuple, "deleted")!!
@@ -134,7 +141,7 @@ class AuftragsCacheService {
                    a.entscheidungsDatum as entscheidungsDatum,a.bemerkung as bemerkung,
                    a.probabilityOfOccurrence as probabilityOfOccurrence,
                    a.periodOfPerformanceBegin as periodOfPerformanceBegin, a.periodOfPerformanceEnd as periodOfPerformanceEnd,
-                   a.contactPerson.id as contactPersonId
+                   a.contactPerson.id as contactPersonId, a.kunde.id as kundeId, a.projekt.id as projektId, a.kundeText as kundeText
             FROM ${AuftragDO::class.simpleName} a
         """.trimIndent()
         private val SELECT_POSITIONS = """

diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt
@@ -32,6 +32,7 @@ import org.projectforge.Constants
 import org.projectforge.business.excel.ExcelDateFormats
 import org.projectforge.business.excel.XlsContentProvider
 import org.projectforge.business.fibu.kost.ProjektCache
+import org.projectforge.business.fibu.orderbooksnapshots.OrderbookSnapshotsService
 import org.projectforge.business.task.TaskTree
 import org.projectforge.business.user.ProjectForgeGroup
 import org.projectforge.common.DateFormatType
@@ -48,6 +49,8 @@ import org.springframework.stereotype.Service
 import java.io.IOException
 import java.math.BigDecimal
 import java.math.RoundingMode
+import java.time.LocalDate
+import java.time.Month
 import java.time.format.DateTimeFormatter
 
 private val log = KotlinLogging.logger {}
@@ -63,6 +66,9 @@ open class ForecastExport { // open needed by Wicket.
     @Autowired
     private lateinit var accessChecker: AccessChecker
 
+    @Autowired
+    private lateinit var orderbookSnapshotsService: OrderbookSnapshotsService
+
     @Autowired
     private lateinit var orderDao: AuftragDao
 
@@ -117,8 +123,9 @@ open class ForecastExport { // open needed by Wicket.
         val forecastSheet: ExcelSheet,
         val invoicesSheet: ExcelSheet,
         val invoicesPriorYearSheet: ExcelSheet,
-        val baseDate: PFDay,
-        val invoices: List<RechnungDO>
+        val startDate: PFDay,
+        val invoices: List<RechnungDO>,
+        val today: PFDay = PFDay.now()
     ) {
         val excelDateFormat =
             ThreadLocalUserContext.loggedInUser?.excelDateFormat ?: ExcelDateFormats.EXCEL_DEFAULT_DATE
@@ -136,7 +143,6 @@ open class ForecastExport { // open needed by Wicket.
             false // showAll is true, if no filter is given and for financial and controlling staff only.
         val orderPositionMap = mutableMapOf<Long, OrderPositionInfo>()
         val orderMapByPositionId = mutableMapOf<Long, OrderInfo>()
-        val today = PFDay.now()
         val thisMonth = today.beginOfMonth
 
         init {
@@ -147,35 +153,57 @@ open class ForecastExport { // open needed by Wicket.
 
     @Throws(IOException::class)
     open fun export(origFilter: AuftragFilter): ByteArray? {
-        val baseDateParam = origFilter.periodOfPerformanceStartDate
-        val baseDate = if (baseDateParam != null) PFDay.from(baseDateParam).beginOfMonth else PFDay.now().beginOfYear
+        val startDateParam = origFilter.periodOfPerformanceStartDate
+        val startDate = if (startDateParam != null) PFDay.from(startDateParam).beginOfMonth else PFDay.now().beginOfYear
         val filter = AuftragFilter()
         filter.searchString = origFilter.searchString
         filter.projectList = origFilter.projectList
         //filter.auftragFakturiertFilterStatus = origFilter.auftragFakturiertFilterStatus
         //filter.auftragsPositionsPaymentType = origFilter.auftragsPositionsPaymentType
         filter.periodOfPerformanceStartDate =
-            baseDate.plusYears(-2).localDate // Go 2 years back for getting all orders referred by invoices of prior year.
+            startDate.plusYears(-2).localDate // Go 2 years back for getting all orders referred by invoices of prior year.
         filter.user = origFilter.user
         val orderList = orderDao.select(filter)
-        log.info { "Exporting forecast script for date ${baseDate.isoString} with filter: str='${filter.searchString ?: ""}', projects=${filter.projectList?.joinToString { it.name ?: "???" }}" }
+        log.info { "Exporting forecast script for date ${startDate.isoString} with filter: str='${filter.searchString ?: ""}', projects=${filter.projectList?.joinToString { it.name ?: "???" }}" }
         val showAll = accessChecker.isLoggedInUserMemberOfGroup(
             ProjectForgeGroup.FINANCE_GROUP,
             ProjectForgeGroup.CONTROLLING_GROUP
         ) &&
                 filter.searchString.isNullOrBlank() &&
                 filter.projectList.isNullOrEmpty()
-        return export(orderList, baseDate, showAll = showAll)
+        return export(orderList, startDate, showAll = showAll)
     }
 
     @Throws(IOException::class)
-    open fun export(orderList: Collection<AuftragDO>, baseDate: PFDay, showAll: Boolean): ByteArray? {
+    open fun export(startDate: PFDay, snapshotDate: PFDay): ByteArray? {
+        val orderList = orderbookSnapshotsService.readSnapshot(snapshotDate.localDate) ?: return null
+        return export(orderList, startDate, showAll = true, snapshotDate)
+    }
+
+    /**
+     * Export the forecast sheet.
+     * @param orderList The list of orders to export.
+     * @param startDate The start date for the forecast.
+     * @param showAll True, if no filter is given, for financial and controlling staff only.
+     * @param today Today (null) or, the day of the snapshot, if the orderList is loaded from order book snapshots.
+     *              If the date is in the past, the forecast will be simulated with the specified date.
+     *              If date is given, no caches will be used.
+     * @return The byte array of the Excel file.
+     */
+    @Throws(IOException::class)
+    open fun export(
+        orderList: Collection<AuftragDO>,
+        startDate: PFDay,
+        showAll: Boolean,
+        today: PFDay? = null,
+    ): ByteArray? {
         if (orderList.isEmpty()) {
             log.info { "No orders found for export." }
             // No orders found, so we don't need the forecast sheet.
             return null
         }
-        val prevYearBaseDate = baseDate.plusYears(-1) // One year back for getting all invoices.
+        val useAuftragsCache = today == null
+        val prevYearBaseDate = startDate.plusYears(-1) // One year back for getting all invoices.
         val invoiceFilter = RechnungFilter()
         invoiceFilter.fromDate =
             prevYearBaseDate.plusDays(-1).localDate // Go 1 day back, paranoia setting for getting all invoices for given time period.
@@ -201,7 +229,15 @@ open class ForecastExport { // open needed by Wicket.
             InvoicesCol.entries.forEach { invoicesPriorYearSheet.registerColumn(it.header) }
             MonthCol.entries.forEach { invoicesPriorYearSheet.registerColumn(it.header) }
 
-            val ctx = Context(workbook, forecastSheet, invoicesSheet, invoicesPriorYearSheet, baseDate, invoices)
+            val ctx = Context(
+                workbook,
+                forecastSheet = forecastSheet,
+                invoicesSheet = invoicesSheet,
+                invoicesPriorYearSheet = invoicesPriorYearSheet,
+                startDate = startDate,
+                invoices = invoices,
+                today = today ?: PFDay.now(),
+            )
             ctx.showAll = showAll
 
             var currentRow = 9
@@ -210,7 +246,7 @@ open class ForecastExport { // open needed by Wicket.
                 auftragDO.projekt?.id?.let { projektId ->
                     ctx.projectIds.add(projektId)
                 }
-                val orderInfo = ordersCache.getOrderInfo(auftragDO)
+                val orderInfo = if (useAuftragsCache) ordersCache.getOrderInfo(auftragDO) else auftragDO.info
                 auftragDO.id?.let { ctx.orderMap[it] = orderInfo }
                 orderInfo.infoPositions?.forEach { pos ->
                     pos.id?.let {
@@ -237,8 +273,8 @@ open class ForecastExport { // open needed by Wicket.
                 return null
             }
             fillInvoices(ctx)
-            replaceMonthDatesInHeaderRow(forecastSheet, baseDate)
-            replaceMonthDatesInHeaderRow(invoicesSheet, baseDate)
+            replaceMonthDatesInHeaderRow(forecastSheet, startDate)
+            replaceMonthDatesInHeaderRow(invoicesSheet, startDate)
             replaceMonthDatesInHeaderRow(invoicesPriorYearSheet, prevYearBaseDate)
             forecastSheet.setAutoFilter()
             invoicesSheet.setAutoFilter()
@@ -386,16 +422,8 @@ open class ForecastExport { // open needed by Wicket.
             ForecastCol.ABRECHNUNGSART.header,
             if (pos.paymentType != null) translate(pos.paymentType.i18nKey) else ""
         )
-        sheet.setStringValue(
-            row,
-            ForecastCol.AUFTRAG_STATUS.header,
-            if (order.status != null) translate(order.status!!.i18nKey) else ""
-        )
-        sheet.setStringValue(
-            row,
-            ForecastCol.POSITION_STATUS.header,
-            if (pos.status != null) translate(pos.status.i18nKey) else ""
-        )
+        sheet.setStringValue(row, ForecastCol.AUFTRAG_STATUS.header, translate(order.status.i18nKey))
+        sheet.setStringValue(row, ForecastCol.POSITION_STATUS.header, translate(pos.status.i18nKey))
         sheet.setIntValue(row, ForecastCol.PT.header, pos.personDays?.toInt() ?: 0)
         sheet.setBigDecimalValue(
             row, ForecastCol.NETTOSUMME.header, pos.netSum
@@ -443,13 +471,13 @@ open class ForecastExport { // open needed by Wicket.
             sheet.setDateValue(
                 row,
                 leistungsZeitraumColDef,
-                PFDay(order.periodOfPerformanceBegin!!).localDate,
+                PFDay.fromOrNull(order.periodOfPerformanceBegin)?.localDate,
                 ctx.excelDateFormat
             )
             sheet.setDateValue(
                 row,
                 leistungsZeitraumColDef.columnNumber + 1,
-                PFDay(order.periodOfPerformanceEnd!!).localDate,
+                PFDay.fromOrNull(order.periodOfPerformanceEnd)?.localDate,
                 ctx.excelDateFormat
             )
         }
@@ -546,7 +574,7 @@ open class ForecastExport { // open needed by Wicket.
         order: OrderInfo, pos: OrderPositionInfo
     ) { // payment values
         val probability = ForecastUtils.getProbabilityOfAccurence(order, pos)
-        var currentMonth = ctx.baseDate.plusMonths(-1).beginOfMonth
+        var currentMonth = ctx.startDate.plusMonths(-1).beginOfMonth
         MonthCol.entries.forEach {
             currentMonth = currentMonth.plusMonths(1)
             if (checkAfterMonthBefore(currentMonth)) {
@@ -589,7 +617,7 @@ open class ForecastExport { // open needed by Wicket.
 
     private fun getMonthIndex(ctx: Context, date: PFDay): Int {
         val monthDate = date.year * 12 + date.monthValue
-        val monthBaseDate = ctx.baseDate.year * 12 + ctx.baseDate.monthValue
+        val monthBaseDate = ctx.startDate.year * 12 + ctx.startDate.monthValue
         return monthDate - monthBaseDate // index from 0 to 11
     }
 

diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastUtils.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastUtils.kt
@@ -76,7 +76,7 @@ object ForecastUtils { // open needed by Wicket.
      */
     @JvmStatic
     fun computeProbabilityNetSum(order: OrderInfo, pos: OrderPositionInfo): BigDecimal {
-        val netSum = if (pos.netSum != null) pos.netSum else BigDecimal.ZERO
+        val netSum = pos.netSum ?: BigDecimal.ZERO
         val probability = getProbabilityOfAccurence(order, pos)
         return netSum.multiply(probability)
     }
@@ -88,13 +88,13 @@ object ForecastUtils { // open needed by Wicket.
     fun getProbabilityOfAccurence(order: OrderInfo, pos: OrderPositionInfo): BigDecimal {
         // See ForecastExportProbabilities.xlsx
         // Excel rows: Order 1-4
-        if (order.status?.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true
-            || pos.status?.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true
+        if (order.status.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true
+            || pos.status.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true
         ) {
             return BigDecimal.ZERO
         }
         // Excel rows: Order 5-6
-        if (pos.status?.isIn(AuftragsStatus.POTENZIAL, AuftragsStatus.OPTIONAL) == true) {
+        if (pos.status.isIn(AuftragsStatus.POTENZIAL, AuftragsStatus.OPTIONAL) == true) {
             return getGivenProbability(order, BigDecimal.ZERO)
         }
         // Excel rows: Order 7
@@ -106,17 +106,17 @@ object ForecastUtils { // open needed by Wicket.
             return getGivenProbability(order, BigDecimal.ZERO)
         }
         // Excel rows: Order 9-10
-        if (order.status?.isIn(AuftragsStatus.ABGESCHLOSSEN, AuftragsStatus.BEAUFTRAGT) == true) {
+        if (order.status.isIn(AuftragsStatus.ABGESCHLOSSEN, AuftragsStatus.BEAUFTRAGT) == true) {
             return BigDecimal.ONE
         }
         // Excel rows: Order 11-12
-        if (order.status?.isIn(
+        if (order.status.isIn(
                 AuftragsStatus.ESKALATION,
                 AuftragsStatus.GELEGT,
                 AuftragsStatus.IN_ERSTELLUNG
             ) == true
         ) {
-            if (pos.status?.isIn(
+            if (pos.status.isIn(
                     AuftragsStatus.ESKALATION,
                     AuftragsStatus.GELEGT,
                     AuftragsStatus.IN_ERSTELLUNG
@@ -131,7 +131,7 @@ object ForecastUtils { // open needed by Wicket.
         }
         // Excel rows: Order 13
         if (order.status == AuftragsStatus.LOI
-            && pos.status?.isIn(
+            && pos.status.isIn(
                 AuftragsStatus.ESKALATION,
                 AuftragsStatus.GELEGT,
                 AuftragsStatus.IN_ERSTELLUNG