From 1fdc2e1e573544942a1f46eb1cc23dbfeb7e133f Mon Sep 17 00:00:00 2001 From: Kai Reinhard Date: Fri, 3 Jan 2025 02:05:03 +0100 Subject: [PATCH 1/2] WIP: OrderbookSnapshots.... --- README.adoc | 2 +- ToDo.adoc | 35 ++++---- .../business/fibu/ForecastExport.kt | 84 ++++++++++++------- .../business/fibu/ForecastUtils.kt | 16 ++-- .../projectforge/business/fibu/OrderInfo.kt | 14 +++- .../business/fibu/OrderPositionInfo.kt | 12 ++- .../business/fibu/orderbooksnapshots/Order.kt | 3 + .../OrderConverterService.kt | 7 ++ .../orderbooksnapshots/OrderbookSnapshotDO.kt | 2 +- .../OrderbookSnapshotsSanityCheck.kt | 8 +- .../OrderbookSnapshotsService.kt | 4 +- 11 files changed, 124 insertions(+), 63 deletions(-) diff --git a/README.adoc b/README.adoc index 6fd890034d..585a7f2963 100644 --- a/README.adoc +++ b/README.adoc @@ -96,7 +96,7 @@ spring.datasource.driver-class-name=org.postgresql.Driver |=== |Stop|`docker stop projectforge-postgres` |Start|`docker start projectforge-postgres` -|Import dump (optional)|`docker run -v ~/ProjectForgeBackup/pf.sql:/mnt/pf.sql -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 psql -h postgres -U projectforge -q -f /mnt/pf.sql 2>&1 > log.txt` +|Import dump (optional)|`docker run -v ~/ProjectForgeBackup/pf.sql:/mnt/pf.sql -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 psql -h postgres -U projectforge -q -f /mnt/pf.sql` |PSQL|`docker run -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 psql -h postgres -U projectforge` |Reset passwords (optional)|`update t_pf_user_password SET password_hash='SHA{BC871652288E56E306CFA093BEFC3FFCD0ED8872}', password_salt=null; update t_pf_user SET password='SHA{BC871652288E56E306CFA093BEFC3FFCD0ED8872}', password_salt=null, email='m.developer@localhost';` + password is now `test123`. diff --git a/ToDo.adoc b/ToDo.adoc index a69b8a033e..52a1c8315f 100644 --- a/ToDo.adoc +++ b/ToDo.adoc @@ -1,5 +1,6 @@ Aktuell: - Scripting: Ergebnis Unresolved reference 'memo', 'todo'.: line 94 to 94 (add only activated plugins) +- Summen in DB-Exporten eintragen und serverseitig evaluieren. - Groovy-scripts: remove or fix. - AG-Grid: setColumnStates wird nicht in den UserPrefs gespeichert. - Wicket: Auftragsbuch: org.apache.wicket.core.request.mapper.StalePageException: A request to page '[Page class = org.projectforge.web.fibu.AuftragEditPage, id = 9, render count = 3]' has been made with stale 'renderCount'. The page will be re-rendered. @@ -25,10 +26,7 @@ Aktuell: org.hibernate.persister.entity.AbstractEntityPersister#generateSelectLazy: History -- Positionen etc. vernünftig anzeigen. - - Suche: - - Suche nach K+S (aktuell noch Provisorium) - HistoryConvertContext: Am Ende alle displayProperties übernehmen. - ProjektEdit: History enthält keine neuen Kost2DOs/Arten (Einträge sind aber da). @@ -43,22 +41,13 @@ History - History of AddressCampaignValueDO's (AddressCampaignValueDao.convertToDisplayHistoryEntries removed) Später -- Hibernate-Search: K+S etc. -- Besuchsbuch und EmployeeValidSinceAttr: DisplayEntries von Besuchtagen mit Datum des Besuchs versehen. +- Fakturaquote - Suche-Seite hat veraltete Bereiche, AddressListPage läuft auf Fehler. - OrderExport: paymentSchedules werden gefetcht. -- Update caches after single load or modification. Ganz später - Kalenderlist ruft x-fach DB: FIND GroupDO resultClass=GroupDO auf. -Postgresql-Dump-Imports bechleunigen: - -ALTER SYSTEM SET fsync = off; -ALTER SYSTEM SET synchronous_commit = off; -SET maintenance_work_mem = '512MB'; -drop view v_t_pf_user; - Rancher docker system df @@ -67,13 +56,23 @@ docker system df docker volume ls docker volume rm -drop table t_fibu_orderbook_storage; -delete from t_flyway_schema_history where version = '2023.11.01'; + +Postgresql-Dump-Imports bechleunigen: + +ALTER SYSTEM SET fsync = off; +ALTER SYSTEM SET synchronous_commit = off; +SET maintenance_work_mem = '512MB'; + + +drop view v_t_pf_user; + Orderbooks importieren: docker cp ~/ProjectForgeBackup/ProjectForge-Orderbook_*.gz projectforge-postgres:/tmp/ \set file_path '/tmp/ProjectForge-Orderbook_2023-11-01.gz' -INSERT INTO t_fibu_orderbook_storage (date, serialized_orderbook) VALUES ('2023-11-01', pg_read_binary_file(:'file_path')::bytea); -\set file_path '/tmp/ProjectForge-Orderbook_2023-12-01.gz' -INSERT INTO t_fibu_orderbook_storage (date, serialized_orderbook) VALUES ('2023-12-01', pg_read_binary_file(:'file_path')::bytea); +INSERT INTO t_fibu_orderbook_snapshots (date, created, serialized_orderbook, size) VALUES ('2023-11-01', NOW(), pg_read_binary_file(:'file_path')::bytea, (pg_stat_file(:'file_path')).size); + +docker run -e PGPASSWORD=$PGPASSWORD -it --rm --link projectforge-postgres:postgres postgres:13.18 pg_dump -h postgres -U projectforge --data-only --column-inserts --table=t_fibu_orderbook_snapshots + +psql -f export.sql diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt index f6e903900a..30483e9f7c 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt @@ -32,6 +32,7 @@ import org.projectforge.Constants import org.projectforge.business.excel.ExcelDateFormats import org.projectforge.business.excel.XlsContentProvider import org.projectforge.business.fibu.kost.ProjektCache +import org.projectforge.business.fibu.orderbooksnapshots.OrderbookSnapshotsService import org.projectforge.business.task.TaskTree import org.projectforge.business.user.ProjectForgeGroup import org.projectforge.common.DateFormatType @@ -48,6 +49,8 @@ import org.springframework.stereotype.Service import java.io.IOException import java.math.BigDecimal import java.math.RoundingMode +import java.time.LocalDate +import java.time.Month import java.time.format.DateTimeFormatter private val log = KotlinLogging.logger {} @@ -63,6 +66,9 @@ open class ForecastExport { // open needed by Wicket. @Autowired private lateinit var accessChecker: AccessChecker + @Autowired + private lateinit var orderbookSnapshotsService: OrderbookSnapshotsService + @Autowired private lateinit var orderDao: AuftragDao @@ -117,8 +123,9 @@ open class ForecastExport { // open needed by Wicket. val forecastSheet: ExcelSheet, val invoicesSheet: ExcelSheet, val invoicesPriorYearSheet: ExcelSheet, - val baseDate: PFDay, - val invoices: List + val startDate: PFDay, + val invoices: List, + val today: PFDay = PFDay.now() ) { val excelDateFormat = ThreadLocalUserContext.loggedInUser?.excelDateFormat ?: ExcelDateFormats.EXCEL_DEFAULT_DATE @@ -136,7 +143,6 @@ open class ForecastExport { // open needed by Wicket. false // showAll is true, if no filter is given and for financial and controlling staff only. val orderPositionMap = mutableMapOf() val orderMapByPositionId = mutableMapOf() - val today = PFDay.now() val thisMonth = today.beginOfMonth init { @@ -147,35 +153,57 @@ open class ForecastExport { // open needed by Wicket. @Throws(IOException::class) open fun export(origFilter: AuftragFilter): ByteArray? { - val baseDateParam = origFilter.periodOfPerformanceStartDate - val baseDate = if (baseDateParam != null) PFDay.from(baseDateParam).beginOfMonth else PFDay.now().beginOfYear + val startDateParam = origFilter.periodOfPerformanceStartDate + val startDate = if (startDateParam != null) PFDay.from(startDateParam).beginOfMonth else PFDay.now().beginOfYear val filter = AuftragFilter() filter.searchString = origFilter.searchString filter.projectList = origFilter.projectList //filter.auftragFakturiertFilterStatus = origFilter.auftragFakturiertFilterStatus //filter.auftragsPositionsPaymentType = origFilter.auftragsPositionsPaymentType filter.periodOfPerformanceStartDate = - baseDate.plusYears(-2).localDate // Go 2 years back for getting all orders referred by invoices of prior year. + startDate.plusYears(-2).localDate // Go 2 years back for getting all orders referred by invoices of prior year. filter.user = origFilter.user val orderList = orderDao.select(filter) - log.info { "Exporting forecast script for date ${baseDate.isoString} with filter: str='${filter.searchString ?: ""}', projects=${filter.projectList?.joinToString { it.name ?: "???" }}" } + log.info { "Exporting forecast script for date ${startDate.isoString} with filter: str='${filter.searchString ?: ""}', projects=${filter.projectList?.joinToString { it.name ?: "???" }}" } val showAll = accessChecker.isLoggedInUserMemberOfGroup( ProjectForgeGroup.FINANCE_GROUP, ProjectForgeGroup.CONTROLLING_GROUP ) && filter.searchString.isNullOrBlank() && filter.projectList.isNullOrEmpty() - return export(orderList, baseDate, showAll = showAll) + return export(orderList, startDate, showAll = showAll) } @Throws(IOException::class) - open fun export(orderList: Collection, baseDate: PFDay, showAll: Boolean): ByteArray? { + open fun export(startDate: PFDay, snapshotDate: PFDay): ByteArray? { + val orderList = orderbookSnapshotsService.readSnapshot(snapshotDate.localDate) ?: return null + return export(orderList, startDate, showAll = true, snapshotDate) + } + + /** + * Export the forecast sheet. + * @param orderList The list of orders to export. + * @param startDate The start date for the forecast. + * @param showAll True, if no filter is given, for financial and controlling staff only. + * @param today Today (null) or, the day of the snapshot, if the orderList is loaded from order book snapshots. + * If the date is in the past, the forecast will be simulated with the specified date. + * If date is given, no caches will be used. + * @return The byte array of the Excel file. + */ + @Throws(IOException::class) + open fun export( + orderList: Collection, + startDate: PFDay, + showAll: Boolean, + today: PFDay? = null, + ): ByteArray? { if (orderList.isEmpty()) { log.info { "No orders found for export." } // No orders found, so we don't need the forecast sheet. return null } - val prevYearBaseDate = baseDate.plusYears(-1) // One year back for getting all invoices. + val useAuftragsCache = today == null + val prevYearBaseDate = startDate.plusYears(-1) // One year back for getting all invoices. val invoiceFilter = RechnungFilter() invoiceFilter.fromDate = prevYearBaseDate.plusDays(-1).localDate // Go 1 day back, paranoia setting for getting all invoices for given time period. @@ -201,7 +229,15 @@ open class ForecastExport { // open needed by Wicket. InvoicesCol.entries.forEach { invoicesPriorYearSheet.registerColumn(it.header) } MonthCol.entries.forEach { invoicesPriorYearSheet.registerColumn(it.header) } - val ctx = Context(workbook, forecastSheet, invoicesSheet, invoicesPriorYearSheet, baseDate, invoices) + val ctx = Context( + workbook, + forecastSheet = forecastSheet, + invoicesSheet = invoicesSheet, + invoicesPriorYearSheet = invoicesPriorYearSheet, + startDate = startDate, + invoices = invoices, + today = today ?: PFDay.now(), + ) ctx.showAll = showAll var currentRow = 9 @@ -210,7 +246,7 @@ open class ForecastExport { // open needed by Wicket. auftragDO.projekt?.id?.let { projektId -> ctx.projectIds.add(projektId) } - val orderInfo = ordersCache.getOrderInfo(auftragDO) + val orderInfo = if (useAuftragsCache) auftragDO.info else ordersCache.getOrderInfo(auftragDO) auftragDO.id?.let { ctx.orderMap[it] = orderInfo } orderInfo.infoPositions?.forEach { pos -> pos.id?.let { @@ -237,8 +273,8 @@ open class ForecastExport { // open needed by Wicket. return null } fillInvoices(ctx) - replaceMonthDatesInHeaderRow(forecastSheet, baseDate) - replaceMonthDatesInHeaderRow(invoicesSheet, baseDate) + replaceMonthDatesInHeaderRow(forecastSheet, startDate) + replaceMonthDatesInHeaderRow(invoicesSheet, startDate) replaceMonthDatesInHeaderRow(invoicesPriorYearSheet, prevYearBaseDate) forecastSheet.setAutoFilter() invoicesSheet.setAutoFilter() @@ -386,16 +422,8 @@ open class ForecastExport { // open needed by Wicket. ForecastCol.ABRECHNUNGSART.header, if (pos.paymentType != null) translate(pos.paymentType.i18nKey) else "" ) - sheet.setStringValue( - row, - ForecastCol.AUFTRAG_STATUS.header, - if (order.status != null) translate(order.status!!.i18nKey) else "" - ) - sheet.setStringValue( - row, - ForecastCol.POSITION_STATUS.header, - if (pos.status != null) translate(pos.status.i18nKey) else "" - ) + sheet.setStringValue(row, ForecastCol.AUFTRAG_STATUS.header, translate(order.status.i18nKey)) + sheet.setStringValue(row, ForecastCol.POSITION_STATUS.header, translate(pos.status.i18nKey)) sheet.setIntValue(row, ForecastCol.PT.header, pos.personDays?.toInt() ?: 0) sheet.setBigDecimalValue( row, ForecastCol.NETTOSUMME.header, pos.netSum @@ -443,13 +471,13 @@ open class ForecastExport { // open needed by Wicket. sheet.setDateValue( row, leistungsZeitraumColDef, - PFDay(order.periodOfPerformanceBegin!!).localDate, + PFDay.fromOrNull(order.periodOfPerformanceBegin)?.localDate, ctx.excelDateFormat ) sheet.setDateValue( row, leistungsZeitraumColDef.columnNumber + 1, - PFDay(order.periodOfPerformanceEnd!!).localDate, + PFDay.fromOrNull(order.periodOfPerformanceEnd)?.localDate, ctx.excelDateFormat ) } @@ -546,7 +574,7 @@ open class ForecastExport { // open needed by Wicket. order: OrderInfo, pos: OrderPositionInfo ) { // payment values val probability = ForecastUtils.getProbabilityOfAccurence(order, pos) - var currentMonth = ctx.baseDate.plusMonths(-1).beginOfMonth + var currentMonth = ctx.startDate.plusMonths(-1).beginOfMonth MonthCol.entries.forEach { currentMonth = currentMonth.plusMonths(1) if (checkAfterMonthBefore(currentMonth)) { @@ -589,7 +617,7 @@ open class ForecastExport { // open needed by Wicket. private fun getMonthIndex(ctx: Context, date: PFDay): Int { val monthDate = date.year * 12 + date.monthValue - val monthBaseDate = ctx.baseDate.year * 12 + ctx.baseDate.monthValue + val monthBaseDate = ctx.startDate.year * 12 + ctx.startDate.monthValue return monthDate - monthBaseDate // index from 0 to 11 } diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastUtils.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastUtils.kt index 90bf3713d3..49e300d8cd 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastUtils.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastUtils.kt @@ -76,7 +76,7 @@ object ForecastUtils { // open needed by Wicket. */ @JvmStatic fun computeProbabilityNetSum(order: OrderInfo, pos: OrderPositionInfo): BigDecimal { - val netSum = if (pos.netSum != null) pos.netSum else BigDecimal.ZERO + val netSum = pos.netSum ?: BigDecimal.ZERO val probability = getProbabilityOfAccurence(order, pos) return netSum.multiply(probability) } @@ -88,13 +88,13 @@ object ForecastUtils { // open needed by Wicket. fun getProbabilityOfAccurence(order: OrderInfo, pos: OrderPositionInfo): BigDecimal { // See ForecastExportProbabilities.xlsx // Excel rows: Order 1-4 - if (order.status?.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true - || pos.status?.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true + if (order.status.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true + || pos.status.isIn(AuftragsStatus.ABGELEHNT, AuftragsStatus.ERSETZT) == true ) { return BigDecimal.ZERO } // Excel rows: Order 5-6 - if (pos.status?.isIn(AuftragsStatus.POTENZIAL, AuftragsStatus.OPTIONAL) == true) { + if (pos.status.isIn(AuftragsStatus.POTENZIAL, AuftragsStatus.OPTIONAL) == true) { return getGivenProbability(order, BigDecimal.ZERO) } // Excel rows: Order 7 @@ -106,17 +106,17 @@ object ForecastUtils { // open needed by Wicket. return getGivenProbability(order, BigDecimal.ZERO) } // Excel rows: Order 9-10 - if (order.status?.isIn(AuftragsStatus.ABGESCHLOSSEN, AuftragsStatus.BEAUFTRAGT) == true) { + if (order.status.isIn(AuftragsStatus.ABGESCHLOSSEN, AuftragsStatus.BEAUFTRAGT) == true) { return BigDecimal.ONE } // Excel rows: Order 11-12 - if (order.status?.isIn( + if (order.status.isIn( AuftragsStatus.ESKALATION, AuftragsStatus.GELEGT, AuftragsStatus.IN_ERSTELLUNG ) == true ) { - if (pos.status?.isIn( + if (pos.status.isIn( AuftragsStatus.ESKALATION, AuftragsStatus.GELEGT, AuftragsStatus.IN_ERSTELLUNG @@ -131,7 +131,7 @@ object ForecastUtils { // open needed by Wicket. } // Excel rows: Order 13 if (order.status == AuftragsStatus.LOI - && pos.status?.isIn( + && pos.status.isIn( AuftragsStatus.ESKALATION, AuftragsStatus.GELEGT, AuftragsStatus.IN_ERSTELLUNG diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt index 216217ee40..03f037babf 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt @@ -79,9 +79,19 @@ class OrderInfo() : Serializable { /** * The positions (not deleted) of the order with additional information. + * If not set (e.g. by OrderbookSnapshot), the positions will be lazy loaded from cache. */ - val infoPositions: Collection? - get() = AuftragsCache.instance.getOrderPositionInfosByAuftragId(id) + var infoPositions: Collection? = null + get() = if (field != null || snapshotVersion) field else AuftragsCache.instance.getOrderPositionInfosByAuftragId(id) + set(value) { + field = value + } + + /** + * If true, the order was loaded from a snapshot. Therefore, no recalculation should be done. + */ + var snapshotVersion: Boolean = false + fun updateFields(order: AuftragDO, paymentSchedules: Collection? = null) { id = order.id diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderPositionInfo.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderPositionInfo.kt index 986ade1c2a..40caac4142 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderPositionInfo.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderPositionInfo.kt @@ -51,7 +51,7 @@ class OrderPositionInfo(position: AuftragsPositionDO, order: OrderInfo) : Serial val dbNetSum = position.nettoSumme ?: BigDecimal.ZERO /** - * For finished postio + * For finished positions. True if the position is fully invoiced and the status is [AuftragsStatus.ABGESCHLOSSEN]. */ val vollstaendigFakturiert = position.vollstaendigFakturiert == true && status == AuftragsStatus.ABGESCHLOSSEN val periodOfPerformanceType = position.periodOfPerformanceType @@ -99,12 +99,20 @@ class OrderPositionInfo(position: AuftragsPositionDO, order: OrderInfo) : Serial */ var notYetInvoiced = BigDecimal.ZERO + /** + * If true, the order position was loaded from a snapshot. Therefore, no recalculation should be done. + */ + var snapshotVersion: Boolean = false + init { /*if (position.status == null) { log.info { "Position without status: $position shouldn't occur. Assuming POTENZIAL." } }*/ if (position.deleted) { - log.debug {"Position is deleted: $position"} + log.debug { "Position is deleted: $position" } + // Nothing to calculate. + } else if (snapshotVersion) { + log.debug { "Position is loaded from snapshot: $position" } // Nothing to calculate. } else { recalculate(order) diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt index ccfbd3754f..c4c27a1ace 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt @@ -28,12 +28,14 @@ import org.projectforge.business.fibu.AuftragsStatus import org.projectforge.business.fibu.OrderInfo import java.math.BigDecimal import java.time.LocalDate +import java.util.Date /** * For storing serializing and deserializing orders. */ internal class Order { var id: Long? = null + var lastUpdate: Date? = null var nummer: Int? = null var positionen: Collection? = null var status: AuftragsStatus? = null @@ -115,6 +117,7 @@ internal class Order { return Order().apply { id = order.id nummer = order.nummer + lastUpdate = order.lastUpdate positionen = order.info.infoPositions?.map { OrderPosition.from(it) } status = order.status kundeId = order.kunde?.id diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt index 6b76149f47..911e9a944b 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt @@ -63,6 +63,7 @@ internal class OrderConverterService { fun from(order: Order): AuftragDO { return AuftragDO().apply { id = order.id + lastUpdate = order.lastUpdate nummer = order.nummer positionen = order.positionen?.map { from(it) }?.toMutableList() status = order.status @@ -74,6 +75,7 @@ internal class OrderConverterService { periodOfPerformanceBegin = order.periodOfPerformanceBegin periodOfPerformanceEnd = order.periodOfPerformanceEnd probabilityOfOccurrence = order.probabilityOfOccurrence + info.nummer = order.nummer info.netSum = order.netSum info.commissionedNetSum = order.commissionedNetSum info.akquiseSum = order.akquiseSum @@ -86,6 +88,11 @@ internal class OrderConverterService { info.positionAbgeschlossenUndNichtVollstaendigFakturiert = order.positionAbgeschlossenUndNichtVollstaendigFakturiert info.paymentSchedulesReached = order.paymentSchedulesReached + info.periodOfPerformanceBegin = order.periodOfPerformanceBegin + info.periodOfPerformanceEnd = order.periodOfPerformanceEnd + info.infoPositions = positionen?.map { OrderPositionInfo(it, info).also { it.snapshotVersion = true } } + info.kundeAsString = kundeAsString + info.projektAsString = projektAsString } } diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotDO.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotDO.kt index d822633947..3eeb5bd75f 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotDO.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotDO.kt @@ -37,7 +37,7 @@ import java.util.* * The purpose of this table is to compare old order books with new ones to compare forecasts with reality. * * ``` - * SELECT date, created, incremental_based_on, octet_length(serialized_orderbook) AS byte_count, size FROM t_fibu_orderbook_snapshots; + * SELECT date, created, incremental_based_on, octet_length(serialized_orderbook) AS byte_count, size FROM t_fibu_orderbook_snapshots order by date desc; * ``` * @author Kai Reinhard (k.reinhard@micromata.de) */ diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsSanityCheck.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsSanityCheck.kt index c2a1c39e93..8be1ee98e7 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsSanityCheck.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsSanityCheck.kt @@ -25,8 +25,10 @@ package org.projectforge.business.fibu.orderbooksnapshots import org.projectforge.common.extensions.format import org.projectforge.common.extensions.formatBytes +import org.projectforge.common.extensions.isoString import org.projectforge.jobs.AbstractJob import org.projectforge.jobs.JobExecutionContext +import java.util.Date class OrderbookSnapshotsSanityCheck(val orderbookSnapshotsService: OrderbookSnapshotsService) : AbstractJob("Checks the recent order book' snapshots.") { @@ -44,8 +46,10 @@ class OrderbookSnapshotsSanityCheck(val orderbookSnapshotsService: OrderbookSnap return@forEach } try { - orderbookSnapshotsService.readSnapshot(date) - jobContext.addMessage("Snapshot for date $date (${it.size.formatBytes()}) is readable.") + val orders = orderbookSnapshotsService.readSnapshot(date) + val lastUpdate = orders?.maxOf { it.lastUpdate ?: Date(0L) } + val highestOrderNumber = orders?.maxOf { it.nummer ?: -1 } + jobContext.addMessage("Snapshot for date $date (${it.size.formatBytes()}) is readable: ${orders?.size?.format()} orders, highest order number=${highestOrderNumber.format()} lastUpdate=${lastUpdate.isoString()}.") } catch (e: Exception) { jobContext.addError("Error reading snapshot for date $date: $e") } diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsService.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsService.kt index f257a95f4a..2be61c1064 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsService.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderbookSnapshotsService.kt @@ -191,7 +191,9 @@ class OrderbookSnapshotsService { fun readSnapshot(date: LocalDate): List? { val orderbook = mutableMapOf() readSnapshot(date, orderbook) - return orderConverterService.convertFromOrder(orderbook.values) + return orderConverterService.convertFromOrder(orderbook.values).also { + log.info { "${it?.size} orders restored from snapshot of $date." } + } } private fun readSnapshot(date: LocalDate, orderbook: MutableMap) { From 7163b8c4e7bf1d7cc4a0dad888a5c75e0d1159fb Mon Sep 17 00:00:00 2001 From: Kai Reinhard Date: Fri, 3 Jan 2025 03:55:07 +0100 Subject: [PATCH 2/2] WIP: Order book snapshots, ForecastExport fixed (customer and project name were missing) --- .../business/fibu/AuftragsCacheService.kt | 13 ++++++++++--- .../projectforge/business/fibu/ForecastExport.kt | 2 +- .../org/projectforge/business/fibu/OrderInfo.kt | 12 +++++++++--- .../business/fibu/orderbooksnapshots/Order.kt | 1 + .../orderbooksnapshots/OrderConverterService.kt | 14 ++++++++++++++ .../fibu/orderbooksnapshots/OrderPosition.kt | 12 ++++++++++++ 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/AuftragsCacheService.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/AuftragsCacheService.kt index c8d7f1c53f..54655de988 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/AuftragsCacheService.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/AuftragsCacheService.kt @@ -66,8 +66,15 @@ class AuftragsCacheService { order.bemerkung = getString(tuple, "bemerkung") order.periodOfPerformanceBegin = getLocalDate(tuple, "periodOfPerformanceBegin") order.periodOfPerformanceEnd = getLocalDate(tuple, "periodOfPerformanceEnd") + order.kundeText = getString(tuple, "kundeText") + getLong(tuple, "kundeId")?.let { kundeId -> + order.kunde = em.getReference(KundeDO::class.java, kundeId) + } + getLong(tuple, "projektId")?.let { projektId -> + order.projekt = em.getReference(ProjektDO::class.java, projektId) + } getLong(tuple, "contactPersonId")?.let { userId -> - order.contactPerson = PFUserDO().also { it.id = userId } + order.contactPerson = em.getReference(PFUserDO::class.java, userId) } } } @@ -82,7 +89,7 @@ class AuftragsCacheService { AuftragsPositionDO().also { pos -> pos.id = getLong(tuple, "id") getLong(tuple, "auftragId")?.let { auftragId -> - pos.auftrag = AuftragDO().also { it.id = auftragId } + pos.auftrag = em.getReference(AuftragDO::class.java, auftragId) } pos.number = getShort(tuple, "number")!! pos.deleted = getBoolean(tuple, "deleted")!! @@ -134,7 +141,7 @@ class AuftragsCacheService { a.entscheidungsDatum as entscheidungsDatum,a.bemerkung as bemerkung, a.probabilityOfOccurrence as probabilityOfOccurrence, a.periodOfPerformanceBegin as periodOfPerformanceBegin, a.periodOfPerformanceEnd as periodOfPerformanceEnd, - a.contactPerson.id as contactPersonId + a.contactPerson.id as contactPersonId, a.kunde.id as kundeId, a.projekt.id as projektId, a.kundeText as kundeText FROM ${AuftragDO::class.simpleName} a """.trimIndent() private val SELECT_POSITIONS = """ diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt index 30483e9f7c..a7c60e7f59 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/ForecastExport.kt @@ -246,7 +246,7 @@ open class ForecastExport { // open needed by Wicket. auftragDO.projekt?.id?.let { projektId -> ctx.projectIds.add(projektId) } - val orderInfo = if (useAuftragsCache) auftragDO.info else ordersCache.getOrderInfo(auftragDO) + val orderInfo = if (useAuftragsCache) ordersCache.getOrderInfo(auftragDO) else auftragDO.info auftragDO.id?.let { ctx.orderMap[it] = orderInfo } orderInfo.infoPositions?.forEach { pos -> pos.id?.let { diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt index 03f037babf..afa9408868 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/OrderInfo.kt @@ -68,8 +68,10 @@ class OrderInfo() : Serializable { var created: Date? = null var erfassungsDatum: LocalDate? = null var entscheidungsDatum: LocalDate? = null - lateinit var kundeAsString: String - lateinit var projektAsString: String + var kundeId: Long? = null + var kundeAsString: String? = null + var projektId: Long? = null + var projektAsString: String? = null var probabilityOfOccurrence: Int? = null var periodOfPerformanceBegin: LocalDate? = null var periodOfPerformanceEnd: LocalDate? = null @@ -82,7 +84,9 @@ class OrderInfo() : Serializable { * If not set (e.g. by OrderbookSnapshot), the positions will be lazy loaded from cache. */ var infoPositions: Collection? = null - get() = if (field != null || snapshotVersion) field else AuftragsCache.instance.getOrderPositionInfosByAuftragId(id) + get() = if (field != null || snapshotVersion) field else AuftragsCache.instance.getOrderPositionInfosByAuftragId( + id + ) set(value) { field = value } @@ -109,7 +113,9 @@ class OrderInfo() : Serializable { created = order.created erfassungsDatum = order.erfassungsDatum entscheidungsDatum = order.entscheidungsDatum + kundeId = order.kunde?.id kundeAsString = order.kundeAsString + projektId = order.projekt?.id projektAsString = order.projektAsString probabilityOfOccurrence = order.probabilityOfOccurrence periodOfPerformanceBegin = order.periodOfPerformanceBegin diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt index c4c27a1ace..92efedbcfb 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/Order.kt @@ -37,6 +37,7 @@ internal class Order { var id: Long? = null var lastUpdate: Date? = null var nummer: Int? = null + var angebotsDatum: LocalDate? = null var positionen: Collection? = null var status: AuftragsStatus? = null var kundeId: Long? = null diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt index 911e9a944b..e8d67f59a1 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderConverterService.kt @@ -28,6 +28,7 @@ import org.projectforge.business.PfCaches import org.projectforge.business.fibu.* import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import java.math.BigDecimal private val log = KotlinLogging.logger {} @@ -65,6 +66,7 @@ internal class OrderConverterService { id = order.id lastUpdate = order.lastUpdate nummer = order.nummer + angebotsDatum = order.angebotsDatum positionen = order.positionen?.map { from(it) }?.toMutableList() status = order.status kunde = caches.getKunde(order.kundeId) @@ -75,7 +77,9 @@ internal class OrderConverterService { periodOfPerformanceBegin = order.periodOfPerformanceBegin periodOfPerformanceEnd = order.periodOfPerformanceEnd probabilityOfOccurrence = order.probabilityOfOccurrence + // Write the fields also to the info object. info.nummer = order.nummer + info.angebotsDatum = order.angebotsDatum info.netSum = order.netSum info.commissionedNetSum = order.commissionedNetSum info.akquiseSum = order.akquiseSum @@ -91,6 +95,16 @@ internal class OrderConverterService { info.periodOfPerformanceBegin = order.periodOfPerformanceBegin info.periodOfPerformanceEnd = order.periodOfPerformanceEnd info.infoPositions = positionen?.map { OrderPositionInfo(it, info).also { it.snapshotVersion = true } } + info.infoPositions?.forEach { infoPos -> + order.positionen?.find { it.number == infoPos.number }?.let { pos -> + infoPos.netSum = pos.netSum ?: BigDecimal.ZERO + infoPos.invoicedSum = pos.invoicedSum ?: BigDecimal.ZERO + infoPos.akquiseSum = pos.akquiseSum ?: BigDecimal.ZERO + infoPos.commissionedNetSum = pos.commissionedNetSum ?: BigDecimal.ZERO + infoPos.notYetInvoiced = pos.notYetInvoiced ?: BigDecimal.ZERO + infoPos.toBeInvoicedSum = pos.toBeInvoicedSum ?: BigDecimal.ZERO + } + } info.kundeAsString = kundeAsString info.projektAsString = projektAsString } diff --git a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderPosition.kt b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderPosition.kt index 39a6e5945a..020a5ea4bc 100644 --- a/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderPosition.kt +++ b/projectforge-business/src/main/kotlin/org/projectforge/business/fibu/orderbooksnapshots/OrderPosition.kt @@ -36,6 +36,12 @@ internal class OrderPosition { var status: AuftragsStatus? = null var titel: String? = null var netSum: BigDecimal? = null + var invoicedSum: BigDecimal? = null + var akquiseSum: BigDecimal? = null + var commissionedNetSum: BigDecimal? = null + var dbNetSum: BigDecimal? = null + var notYetInvoiced: BigDecimal? = null + var toBeInvoicedSum: BigDecimal? = null var personDays: BigDecimal? = null var vollstaendigFakturiert: Boolean? = false var periodOfPerformanceType: PeriodOfPerformanceType? = PeriodOfPerformanceType.SEEABOVE @@ -54,6 +60,12 @@ internal class OrderPosition { status = pos.status titel = Order.abbreviate(pos.titel) netSum = pos.netSum + invoicedSum = pos.invoicedSum + akquiseSum = pos.akquiseSum + commissionedNetSum = pos.commissionedNetSum + dbNetSum = pos.dbNetSum + notYetInvoiced = pos.notYetInvoiced + toBeInvoicedSum = pos.toBeInvoicedSum personDays = pos.personDays vollstaendigFakturiert = pos.vollstaendigFakturiert periodOfPerformanceType = pos.periodOfPerformanceType