From f7d583227a2ca1245fad93976ecd39f6c7e0533d Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Tue, 29 Apr 2025 12:33:28 +0200 Subject: [PATCH 1/7] prepare olap data query --- common/model/query/query/keys.js | 1 + common/model/query/query/query.js | 1 + webapp/components/DataQuery/ButtonBar/ButtonBar.js | 14 ++++++++------ .../QueryNodeDefsSelector/QueryNodeDefsSelector.js | 11 +++++++---- .../NodeDefsSelector/NodeDefsSelectorAggregate.js | 9 +++++++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/common/model/query/query/keys.js b/common/model/query/query/keys.js index efb4c7c8e3..432d6fa617 100644 --- a/common/model/query/query/keys.js +++ b/common/model/query/query/keys.js @@ -15,6 +15,7 @@ export const modes = { raw: 'raw', rawEdit: 'rawEdit', aggregate: 'aggregate', + olap: 'olap', } export const displayTypes = { diff --git a/common/model/query/query/query.js b/common/model/query/query/query.js index a6874fb5f3..7a3971d0eb 100644 --- a/common/model/query/query/query.js +++ b/common/model/query/query/query.js @@ -47,6 +47,7 @@ const isMode = (mode) => (query) => getMode(query) === mode export const isModeAggregate = isMode(modes.aggregate) export const isModeRaw = isMode(modes.raw) export const isModeRawEdit = isMode(modes.rawEdit) +export const isModeOlap = isMode(modes.olap) // utils export const hasSelection = (query) => diff --git a/webapp/components/DataQuery/ButtonBar/ButtonBar.js b/webapp/components/DataQuery/ButtonBar/ButtonBar.js index f2804d239f..659e7e1396 100644 --- a/webapp/components/DataQuery/ButtonBar/ButtonBar.js +++ b/webapp/components/DataQuery/ButtonBar/ButtonBar.js @@ -43,14 +43,16 @@ const modeButtonItems = [ iconClassName: 'icon-sigma', label: 'dataView.dataQuery.mode.aggregate', }, + { + key: modes.olap, + iconClassName: 'icon-sigma', + label: 'dataView.dataQuery.mode.olap', + }, ] -const uiModeByQueryMode = { - [modes.raw]: modes.raw, +const getUiModeByQueryMode = (mode) => // raw edit mode shown as "raw" in mode button group - [modes.rawEdit]: modes.raw, - [modes.aggregate]: modes.aggregate, -} + mode === modes.rawEdit ? modes.raw : mode const ButtonBar = (props) => { const { @@ -81,7 +83,7 @@ const ButtonBar = (props) => { }, [onChangeQuery, query] ) - const selectedMode = uiModeByQueryMode[Query.getMode(query)] + const selectedMode = getUiModeByQueryMode(Query.getMode(query)) const modeEdit = Query.isModeRawEdit(query) const hasSelection = Query.hasSelection(query) const queryChangeDisabled = modeEdit || !dataLoaded || dataLoading diff --git a/webapp/components/DataQuery/QueryNodeDefsSelector/QueryNodeDefsSelector.js b/webapp/components/DataQuery/QueryNodeDefsSelector/QueryNodeDefsSelector.js index 97e0a62cac..18b6b6b992 100644 --- a/webapp/components/DataQuery/QueryNodeDefsSelector/QueryNodeDefsSelector.js +++ b/webapp/components/DataQuery/QueryNodeDefsSelector/QueryNodeDefsSelector.js @@ -16,6 +16,8 @@ const QueryNodeDefsSelector = (props) => { const query = DataExplorerSelectors.useQuery() const modeAggregate = Query.isModeAggregate(query) + const modeOlap = Query.isModeOlap(query) + const mode = Query.getMode(query) const survey = useSurvey() const hierarchy = Survey.getHierarchy(NodeDef.isEntityOrMultiple)(survey) @@ -24,15 +26,15 @@ const QueryNodeDefsSelector = (props) => { const onChangeEntity = useCallback( (entityDefUuid) => { let newQuery = Query.create({ entityDefUuid }) - if (modeAggregate) { - newQuery = Query.assocMode(Query.modes.aggregate)(newQuery) + if (modeAggregate || modeOlap) { + newQuery = Query.assocMode(mode)(newQuery) } onChangeQuery(newQuery) }, - [modeAggregate, onChangeQuery] + [mode, modeAggregate, modeOlap, onChangeQuery] ) - return modeAggregate ? ( + return modeAggregate || modeOlap ? ( { onChangeEntity={onChangeEntity} onChangeMeasures={(measuresUpdate) => onChangeQuery(Query.assocMeasures(measuresUpdate)(query))} onChangeDimensions={(dimensionsUpdate) => onChangeQuery(Query.assocDimensions(dimensionsUpdate)(query))} + olap={modeOlap} showAnalysisAttributes /> ) : ( diff --git a/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js b/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js index af4410ae0d..6eb58468ff 100644 --- a/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js +++ b/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js @@ -39,6 +39,7 @@ const NodeDefsSelectorAggregate = (props) => { measures, nodeDefLabelType = NodeDef.NodeDefLabelTypes.label, nodeDefUuidEntity = null, + olap = false, onChangeEntity, onChangeMeasures, onChangeDimensions, @@ -96,7 +97,10 @@ const NodeDefsSelectorAggregate = (props) => { - NodeDef.isCode(nodeDef) || NodeDef.isTaxon(nodeDef) || NodeDef.isKey(nodeDef) + NodeDef.isKey(nodeDef) || + NodeDef.isCode(nodeDef) || + NodeDef.isTaxon(nodeDef) || + (olap && NodeDef.isBoolean(nodeDef)) } nodeDefLabelType={nodeDefLabelType} nodeDefUuidEntity={nodeDefUuidEntity} @@ -111,7 +115,7 @@ const NodeDefsSelectorAggregate = (props) => { !NodeDef.isKey(nodeDef)} + filterFunction={(nodeDef) => !NodeDef.isKey(nodeDef) && (!olap || NodeDef.isAnalysis(nodeDef))} includeEntityFrequencySelector nodeDefLabelType={nodeDefLabelType} nodeDefUuidEntity={nodeDefUuidEntity} @@ -152,6 +156,7 @@ NodeDefsSelectorAggregate.propTypes = { measures: PropTypes.object.isRequired, nodeDefLabelType: PropTypes.string, nodeDefUuidEntity: PropTypes.string, + olap: PropTypes.bool, onChangeEntity: PropTypes.func.isRequired, onChangeMeasures: PropTypes.func.isRequired, onChangeDimensions: PropTypes.func.isRequired, From 6b6c7bd7a6ab0fb95403bb8c7b53da7c37328c6b Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:51:57 +0200 Subject: [PATCH 2/7] create olap data area view --- .../model/db/tables/olapData/olapAreaView.js | 25 +++++++++++++++++++ common/model/db/tables/olapData/table.js | 4 +++ core/survey/nodeDef.js | 16 +++++++----- .../surveyRdb/manager/surveyRdbManager.js | 1 + .../repository/olapAreaView/create.js | 20 +++++++++++++++ .../repository/olapAreaView/index.js | 1 + .../surveyRdbOlapDataTablesCreationJob.js | 2 ++ .../NodeDefsSelectorAggregate.js | 2 +- 8 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 common/model/db/tables/olapData/olapAreaView.js create mode 100644 server/modules/surveyRdb/repository/olapAreaView/create.js create mode 100644 server/modules/surveyRdb/repository/olapAreaView/index.js diff --git a/common/model/db/tables/olapData/olapAreaView.js b/common/model/db/tables/olapData/olapAreaView.js new file mode 100644 index 0000000000..e186785340 --- /dev/null +++ b/common/model/db/tables/olapData/olapAreaView.js @@ -0,0 +1,25 @@ +import * as Survey from '@core/survey/survey' +import * as NodeDef from '@core/survey/nodeDef' + +import TableSurveyRdb from '../tableSurveyRdb' +import TableOlapData from './table' + +const generateViewName = ({ survey, cycle, entityDef, baseUnitDef }) => { + const dataTable = new TableOlapData({ survey, cycle, entityDef, baseUnitDef }) + return dataTable.name + '_area_view' +} + +export default class OlapAreaView extends TableSurveyRdb { + constructor({ survey, cycle, entityDef, baseUnitDef }) { + super(Survey.getId(survey), generateViewName({ survey, cycle, entityDef, baseUnitDef })) + this._baseUnitDef = baseUnitDef + } + + get baseUnitUuidColumnName() { + return NodeDef.getName(this._baseUnitDef) + '_uuid' + } + + get areaColumnName() { + return 'area' + } +} diff --git a/common/model/db/tables/olapData/table.js b/common/model/db/tables/olapData/table.js index e9d2028f7c..545edffd40 100644 --- a/common/model/db/tables/olapData/table.js +++ b/common/model/db/tables/olapData/table.js @@ -80,6 +80,10 @@ export default class TableOlapData extends TableSurveyRdb { return NodeDef.getName(this._baseUnitDef) + '_uuid' } + get expFactorColumnName() { + return baseColumnSet.expFactor + } + get columnNamesAndTypes() { return [ // base columns diff --git a/core/survey/nodeDef.js b/core/survey/nodeDef.js index 8db4d08f63..e7f8c57053 100644 --- a/core/survey/nodeDef.js +++ b/core/survey/nodeDef.js @@ -307,7 +307,13 @@ export const getMeta = R.propOr({}, keys.meta) export const getMetaHierarchy = R.pathOr([], [keys.meta, metaKeys.h]) // Utils -export const getLabel = (nodeDef, lang, type = NodeDefLabelTypes.label, defaultToName = true) => { +const getLabelSuffix = (nodeDef) => { + if (isVirtual(nodeDef)) return ' (V)' + if (isAnalysis(nodeDef) && isAttribute(nodeDef)) return ' (C)' + return '' +} + +export const getLabel = (nodeDef, lang, type = NodeDefLabelTypes.label, defaultToName = true, includeSuffix = true) => { let firstPart = '' const name = getName(nodeDef) @@ -328,12 +334,10 @@ export const getLabel = (nodeDef, lang, type = NodeDefLabelTypes.label, defaultT if (StringUtils.isBlank(firstPart) && defaultToName) { firstPart = name } - - const suffix = isVirtual(nodeDef) ? ' (V)' : isAnalysis(nodeDef) && isAttribute(nodeDef) ? ' (C)' : '' - - return firstPart + suffix + return firstPart + (includeSuffix ? getLabelSuffix(nodeDef) : '') } -export const getLabelWithType = ({ nodeDef, lang, type }) => getLabel(nodeDef, lang, type) +export const getLabelWithType = ({ nodeDef, lang, type, includeSuffix }) => + getLabel(nodeDef, lang, type, true, includeSuffix) export const getDescription = (lang) => (nodeDef) => R.propOr('', lang, getDescriptions(nodeDef)) diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js index ae6d21d722..857fcd02d3 100644 --- a/server/modules/surveyRdb/manager/surveyRdbManager.js +++ b/server/modules/surveyRdb/manager/surveyRdbManager.js @@ -54,6 +54,7 @@ export { createNodeKeysHierarchyView } from '../repository/nodeKeysHierarchyView export { deleteNodeResultsByChainUuid, MassiveUpdateData, MassiveUpdateNodes } from '../repository/resultNode' export { createOlapDataTable, insertOlapData, clearOlapData } from '../repository/olapDataTable' +export { createOlapAreaView } from '../repository/olapAreaView' // ==== DML diff --git a/server/modules/surveyRdb/repository/olapAreaView/create.js b/server/modules/surveyRdb/repository/olapAreaView/create.js new file mode 100644 index 0000000000..e43fe7888a --- /dev/null +++ b/server/modules/surveyRdb/repository/olapAreaView/create.js @@ -0,0 +1,20 @@ +import OlapAreaView from '@common/model/db/tables/olapData/olapAreaView' +import TableOlapData from '@common/model/db/tables/olapData/table' + +import { db } from '@server/db/db' + +export const createOlapAreaView = async ({ survey, cycle, baseUnitDef, entityDef }, client = db) => { + const view = new OlapAreaView({ survey, cycle, baseUnitDef, entityDef }) + const table = new TableOlapData({ survey, cycle, baseUnitDef, entityDef }) + + return client.query( + `CREATE OR REPLACE VIEW + ${view.nameQualified} + AS ( + SELECT DISTINCT (${table.baseUnitUuidColumnName}) AS ${view.baseUnitUuidColumnName}, + ${table.expFactorColumnName} AS ${view.areaColumnName} + FROM + ${table.nameQualified} + )` + ) +} diff --git a/server/modules/surveyRdb/repository/olapAreaView/index.js b/server/modules/surveyRdb/repository/olapAreaView/index.js new file mode 100644 index 0000000000..b1d54883b0 --- /dev/null +++ b/server/modules/surveyRdb/repository/olapAreaView/index.js @@ -0,0 +1 @@ +export { createOlapAreaView } from './create' diff --git a/server/modules/surveyRdb/service/surveyRdbCreationJob/surveyRdbOlapDataTablesCreationJob.js b/server/modules/surveyRdb/service/surveyRdbCreationJob/surveyRdbOlapDataTablesCreationJob.js index a5b093b21b..6dd3da9c44 100644 --- a/server/modules/surveyRdb/service/surveyRdbCreationJob/surveyRdbOlapDataTablesCreationJob.js +++ b/server/modules/surveyRdb/service/surveyRdbCreationJob/surveyRdbOlapDataTablesCreationJob.js @@ -137,6 +137,8 @@ export default class SurveyRdbOlapDataTablesCreationJob extends Job { async (entityDef) => { this.logDebug(`create OLAP table for entity def ${NodeDef.getName(entityDef)}`) await SurveyRdbManager.createOlapDataTable({ survey, cycle, baseUnitDef, entityDef }, tx) + this.logDebug(`create OLAP area view for entity def ${NodeDef.getName(entityDef)}`) + await SurveyRdbManager.createOlapAreaView({ survey, cycle, baseUnitDef, entityDef }, tx) this.incrementProcessedItems() }, stopIfFunction diff --git a/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js b/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js index 6eb58468ff..80fd99e204 100644 --- a/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js +++ b/webapp/components/survey/NodeDefsSelector/NodeDefsSelectorAggregate.js @@ -116,7 +116,7 @@ const NodeDefsSelectorAggregate = (props) => { !NodeDef.isKey(nodeDef) && (!olap || NodeDef.isAnalysis(nodeDef))} - includeEntityFrequencySelector + includeEntityFrequencySelector={!olap} nodeDefLabelType={nodeDefLabelType} nodeDefUuidEntity={nodeDefUuidEntity} nodeDefUuidsAttributes={measuresNodeDefUuids} From 8a6ffff98a51a3e6c00d0c314a4df5cffa296d81 Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Wed, 30 Apr 2025 12:24:47 +0200 Subject: [PATCH 3/7] prepare olap table select --- common/model/db/tables/olapData/table.js | 6 +++- .../repository/olapDataTable/select.js | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 server/modules/surveyRdb/repository/olapDataTable/select.js diff --git a/common/model/db/tables/olapData/table.js b/common/model/db/tables/olapData/table.js index 545edffd40..0a6a7e8018 100644 --- a/common/model/db/tables/olapData/table.js +++ b/common/model/db/tables/olapData/table.js @@ -29,6 +29,10 @@ export default class TableOlapData extends TableSurveyRdb { this._baseUnitDef = baseUnitDef } + getColumnNameByAttributeDef = (attributeDef) => { + return NodeDef.getName(attributeDef) + } + get attributeDefsForColumns() { const attributeDefs = [] Survey.visitAncestorsAndSelf(this._entityDef, (ancestorDef) => { @@ -57,7 +61,7 @@ export default class TableOlapData extends TableSurveyRdb { // base unit UUID this.baseUnitUuidColumnName, // attribute columns - ...this.attributeDefsForColumns.map(NodeDef.getName), + ...this.attributeDefsForColumns.map((attributeDef) => this.getColumnNameByAttributeDef(attributeDef)), ] } diff --git a/server/modules/surveyRdb/repository/olapDataTable/select.js b/server/modules/surveyRdb/repository/olapDataTable/select.js new file mode 100644 index 0000000000..2f7f328464 --- /dev/null +++ b/server/modules/surveyRdb/repository/olapDataTable/select.js @@ -0,0 +1,28 @@ +import OlapAreaView from '@common/model/db/tables/olapData/olapAreaView' +import TableOlapData from '@common/model/db/tables/olapData/table' + +import * as Survey from '@core/survey/survey' +import * as NodeDef from '@core/survey/nodeDef' + +export const selectAreaFromOlapDataTable = async ( + { survey, cycle, baseUnitDef, entityDef, dimensionsDefs, measuresDefs }, + client = db +) => { + const table = new TableOlapData({ survey, cycle, baseUnitDef, entityDef }) + const dimensionColumnNames = dimensionsDefs.map((dimensionDef) => table.getColumnNameByAttributeDef(dimensionDef)) + const dimensionColumnNamesJoint = dimensionColumnNames.join(', ') + const measuresColumnNames = measuresDefs.map((measureDef) => table.getColumnNameByAttributeDef(measureDef)) + const measuresSelector = measuresColumnNames.map( + (measureColumnName) => `SUM(${measureColumnName}) AS ${measureColumnName}` + ) + const measuresSelectorHa = measuresColumnNames.map( + (measureColumnName) => `SUM(${measureColumnName})/area AS ${measureColumnName}_ha` + ) + + client.query(`WITH area_table AS ( + SELECT ${dimensionColumnNamesJoint}, SUM(${table.expFactorColumnName}) AS area FROM ( + SELECT DISTINCT(${table.baseUnitUuidColumnName}), ${table.expFactorColumnName}, ${dimensionColumnNamesJoint} + FROM ${table.nameQualified} + ) GROUP BY ${dimensionColumnNamesJoint} + )`) +} From 012cb24041c6d66990ef64ed7702b70044ec25fc Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Wed, 30 Apr 2025 12:24:58 +0200 Subject: [PATCH 4/7] prepare olap table select --- server/modules/surveyRdb/repository/olapDataTable/select.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/modules/surveyRdb/repository/olapDataTable/select.js b/server/modules/surveyRdb/repository/olapDataTable/select.js index 2f7f328464..c725f15d61 100644 --- a/server/modules/surveyRdb/repository/olapDataTable/select.js +++ b/server/modules/surveyRdb/repository/olapDataTable/select.js @@ -4,6 +4,8 @@ import TableOlapData from '@common/model/db/tables/olapData/table' import * as Survey from '@core/survey/survey' import * as NodeDef from '@core/survey/nodeDef' +import { db } from '@server/db/db' + export const selectAreaFromOlapDataTable = async ( { survey, cycle, baseUnitDef, entityDef, dimensionsDefs, measuresDefs }, client = db From 9510200135f4b86307296c9f90f9bc633cae1dad Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:53:59 +0200 Subject: [PATCH 5/7] improved olap data table results query (WIP) --- .../surveyRdb/repository/olapDataTable/select.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/modules/surveyRdb/repository/olapDataTable/select.js b/server/modules/surveyRdb/repository/olapDataTable/select.js index c725f15d61..818bd4fe0c 100644 --- a/server/modules/surveyRdb/repository/olapDataTable/select.js +++ b/server/modules/surveyRdb/repository/olapDataTable/select.js @@ -17,14 +17,20 @@ export const selectAreaFromOlapDataTable = async ( const measuresSelector = measuresColumnNames.map( (measureColumnName) => `SUM(${measureColumnName}) AS ${measureColumnName}` ) + const measuresSelectorJoint = measuresSelector.join(', ') const measuresSelectorHa = measuresColumnNames.map( (measureColumnName) => `SUM(${measureColumnName})/area AS ${measureColumnName}_ha` ) + const measuresSelectorHaJoint = measuresSelectorHa.join(', ') - client.query(`WITH area_table AS ( - SELECT ${dimensionColumnNamesJoint}, SUM(${table.expFactorColumnName}) AS area FROM ( + const areaTableSelect = `SELECT ${dimensionColumnNamesJoint}, SUM(${table.expFactorColumnName}) AS area FROM ( SELECT DISTINCT(${table.baseUnitUuidColumnName}), ${table.expFactorColumnName}, ${dimensionColumnNamesJoint} FROM ${table.nameQualified} - ) GROUP BY ${dimensionColumnNamesJoint} - )`) + ) GROUP BY ${dimensionColumnNamesJoint}` + + client.query(`WITH area_table AS (${areaTableSelect}) + + SELECT ${table.baseUnitUuidColumnName}, ${dimensionColumnNamesJoint}, ${measuresSelectorJoint}, ${measuresSelectorHaJoint} + FROM ${table.nameQualified} + GROUP BY ${dimensionColumnNamesJoint}, ${measuresSelectorJoint}`) } From 24ead966b638a38ce63f42f433452ee91e409471 Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Mon, 28 Jul 2025 14:13:30 +0200 Subject: [PATCH 6/7] prepare olap table select --- common/model/db/sql/sqlSelectOlapBuilder.js | 39 ++++++++++++ core/i18n/resources/en/common.js | 1 + .../surveyRdb/manager/surveyRdbManager.js | 19 ++++++ .../repository/olapDataTable/read.js | 62 +++++++++++++++++++ .../repository/olapDataTable/select.js | 36 ----------- .../surveyRdb/service/surveyRdbService.js | 52 ++++++++-------- 6 files changed, 148 insertions(+), 61 deletions(-) create mode 100644 common/model/db/sql/sqlSelectOlapBuilder.js create mode 100644 server/modules/surveyRdb/repository/olapDataTable/read.js delete mode 100644 server/modules/surveyRdb/repository/olapDataTable/select.js diff --git a/common/model/db/sql/sqlSelectOlapBuilder.js b/common/model/db/sql/sqlSelectOlapBuilder.js new file mode 100644 index 0000000000..7bf1a36d37 --- /dev/null +++ b/common/model/db/sql/sqlSelectOlapBuilder.js @@ -0,0 +1,39 @@ +import * as Survey from '@core/survey/survey' + +import SqlSelectBuilder from './sqlSelectBuilder' + +class SqlSelectOlapBuilder extends SqlSelectBuilder { + constructor({ table, entityDef }) { + super() + this._table = table + this._entityDef = entityDef + } + + _selectNodeDef({ nodeDefUuid }) { + const { _table: table } = this + const { survey } = table + const nodeDef = Survey.getNodeDefByUuid(nodeDefUuid)(survey) + const columnName = table.getColumnNameByAttributeDef(nodeDef) + this.select(columnName) + return this + } + + selectMeasure({ nodeDefUuid }) { + const { _table: table } = this + const { survey } = table + const nodeDef = Survey.getNodeDefByUuid(nodeDefUuid)(survey) + const columnName = table.getColumnNameByAttributeDef(nodeDef) + this.select(`SUM(${columnName} AS ${columnName}`) + this.select(`SUM(${columnName})/${SqlSelectOlapBuilder.areaAlias} AS ${columnName}_ha`) + this.groupBy(columnName) + return this + } + + selectDimension({ nodeDefUuid }) { + return this._selectNodeDef({ nodeDefUuid }) + } +} + +SqlSelectOlapBuilder.areaAlias = 'area' + +export default SqlSelectOlapBuilder diff --git a/core/i18n/resources/en/common.js b/core/i18n/resources/en/common.js index 3d30d8bb46..9c34195211 100644 --- a/core/i18n/resources/en/common.js +++ b/core/i18n/resources/en/common.js @@ -749,6 +749,7 @@ Please refine your query (e.g. adding a filter) to reduce the number of items. mode: { label: 'Mode:', aggregate: 'Aggregate', + olap: 'OLAP', raw: 'Raw', rawEdit: 'Raw edit', }, diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js index e677c733d4..bf2731685b 100644 --- a/server/modules/surveyRdb/manager/surveyRdbManager.js +++ b/server/modules/surveyRdb/manager/surveyRdbManager.js @@ -204,6 +204,25 @@ export const fetchViewDataAgg = async (params, client = db) => { return result } +export const fetchOlapData = async (params, client = db) => { + const { survey, cycle, query, recordOwnerUuid = null, limit, offset, outputStream = null, options = {} } = params + const { fileFormat = FileFormats.csv } = options + + // Fetch data + const result = await DataViewRepository.fetchViewDataAgg( + { + survey, + cycle, + query, + recordOwnerUuid, + limit, + offset, + stream: Boolean(outputStream), + }, + client + ) +} + const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuids, search }) => { if (recordUuids) return recordUuids diff --git a/server/modules/surveyRdb/repository/olapDataTable/read.js b/server/modules/surveyRdb/repository/olapDataTable/read.js new file mode 100644 index 0000000000..6448609374 --- /dev/null +++ b/server/modules/surveyRdb/repository/olapDataTable/read.js @@ -0,0 +1,62 @@ +import { Objects } from '@openforis/arena-core' + +import SqlSelectOlapBuilder from '@common/model/db/sql/sqlSelectOlapBuilder' +import TableOlapData from '@common/model/db/tables/olapData/table' +import { Query, Sort } from '@common/model/query' + +import * as Survey from '@core/survey/survey' + +import { db } from '@server/db/db' + +const _getSelectQuery = ({ survey, cycle, query }) => { + const entityDefUuid = Query.getEntityDefUuid(query) + + const entityDef = Survey.getNodeDefByUuid(entityDefUuid)(survey) + const table = new TableOlapData({ survey, cycle, entityDef }) + + const queryBuilder = new SqlSelectOlapBuilder({ table, entityDef }) + + // base unit uuid + queryBuilder.select(table.baseUnitUuidColumnName) + + // SELECT dimensions + Query.getDimensions(query).forEach((nodeDefUuid) => queryBuilder.selectDimension({ nodeDefUuid })) + + // SELECT measures + Object.entries(Query.getMeasures(query)).forEach(([nodeDefUuid]) => queryBuilder.selectMeasure({ nodeDefUuid })) + + // FROM clause + queryBuilder.from(table.nameQualified) + + // ORDER BY clause + const sort = Query.getSort(query) + const { clause: sortClause, params: sortParams } = Sort.toSql(sort) + if (Objects.isNotEmpty(sortParams)) { + queryBuilder.orderBy(sortClause) + queryBuilder.addParams(sortParams) + } + return { select: queryBuilder.build(), queryParams: queryBuilder.params } +} + +export const selectFromOlapDataTable = async ({ survey, cycle, query, baseUnitDef, entityDef }, client = db) => { + const { select, queryParams } = _getSelectQuery({ survey, cycle, query }) + + const table = new TableOlapData({ survey, cycle, baseUnitDef, entityDef }) + const dimensionUuids = Query.getDimensions(query) + const dimensionColumnNames = dimensionUuids.map((dimensionUuid) => { + const dimensionDef = Survey.getNodeDefByUuid(dimensionUuid)(survey) + return table.getColumnNameByAttributeDef(dimensionDef) + }) + const dimensionColumnNamesJoint = dimensionColumnNames.join(', ') + + const areaTableSelect = `SELECT ${dimensionColumnNamesJoint}, SUM(${table.expFactorColumnName}) AS ${SqlSelectOlapBuilder.areaAlias} FROM ( + SELECT DISTINCT(${table.baseUnitUuidColumnName}), ${table.expFactorColumnName}, ${dimensionColumnNamesJoint} + FROM ${table.nameQualified} + ) GROUP BY ${dimensionColumnNamesJoint}` + + return client.query( + `WITH area_table AS (${areaTableSelect}) + ${select}`, + queryParams + ) +} diff --git a/server/modules/surveyRdb/repository/olapDataTable/select.js b/server/modules/surveyRdb/repository/olapDataTable/select.js deleted file mode 100644 index 818bd4fe0c..0000000000 --- a/server/modules/surveyRdb/repository/olapDataTable/select.js +++ /dev/null @@ -1,36 +0,0 @@ -import OlapAreaView from '@common/model/db/tables/olapData/olapAreaView' -import TableOlapData from '@common/model/db/tables/olapData/table' - -import * as Survey from '@core/survey/survey' -import * as NodeDef from '@core/survey/nodeDef' - -import { db } from '@server/db/db' - -export const selectAreaFromOlapDataTable = async ( - { survey, cycle, baseUnitDef, entityDef, dimensionsDefs, measuresDefs }, - client = db -) => { - const table = new TableOlapData({ survey, cycle, baseUnitDef, entityDef }) - const dimensionColumnNames = dimensionsDefs.map((dimensionDef) => table.getColumnNameByAttributeDef(dimensionDef)) - const dimensionColumnNamesJoint = dimensionColumnNames.join(', ') - const measuresColumnNames = measuresDefs.map((measureDef) => table.getColumnNameByAttributeDef(measureDef)) - const measuresSelector = measuresColumnNames.map( - (measureColumnName) => `SUM(${measureColumnName}) AS ${measureColumnName}` - ) - const measuresSelectorJoint = measuresSelector.join(', ') - const measuresSelectorHa = measuresColumnNames.map( - (measureColumnName) => `SUM(${measureColumnName})/area AS ${measureColumnName}_ha` - ) - const measuresSelectorHaJoint = measuresSelectorHa.join(', ') - - const areaTableSelect = `SELECT ${dimensionColumnNamesJoint}, SUM(${table.expFactorColumnName}) AS area FROM ( - SELECT DISTINCT(${table.baseUnitUuidColumnName}), ${table.expFactorColumnName}, ${dimensionColumnNamesJoint} - FROM ${table.nameQualified} - ) GROUP BY ${dimensionColumnNamesJoint}` - - client.query(`WITH area_table AS (${areaTableSelect}) - - SELECT ${table.baseUnitUuidColumnName}, ${dimensionColumnNamesJoint}, ${measuresSelectorJoint}, ${measuresSelectorHaJoint} - FROM ${table.nameQualified} - GROUP BY ${dimensionColumnNamesJoint}, ${measuresSelectorJoint}`) -} diff --git a/server/modules/surveyRdb/service/surveyRdbService.js b/server/modules/surveyRdb/service/surveyRdbService.js index 41d33e7f53..e710db4b33 100644 --- a/server/modules/surveyRdb/service/surveyRdbService.js +++ b/server/modules/surveyRdb/service/surveyRdbService.js @@ -70,31 +70,33 @@ export const fetchViewData = async (params) => { const survey = await _fetchSurvey({ surveyId, cycle }) const recordOwnerUuid = _getRecordOwnerUuidForQuery({ user, survey }) - const data = Query.isModeAggregate(parsedQuery) - ? await SurveyRdbManager.fetchViewDataAgg({ - survey, - cycle, - query, - recordOwnerUuid, - offset, - limit, - outputStream, - options, - }) - : await SurveyRdbManager.fetchViewData({ - survey, - cycle, - query, - columnNodeDefs, - recordOwnerUuid, - offset, - limit, - outputStream, - addCycle, - ...options, - }) - - return data + if (Query.isModeAggregate(parsedQuery)) { + return SurveyRdbManager.fetchViewDataAgg({ + survey, + cycle, + query, + recordOwnerUuid, + offset, + limit, + outputStream, + options, + }) + } + if (Query.isModeOlap(parsedQuery)) { + // TODO + } + return SurveyRdbManager.fetchViewData({ + survey, + cycle, + query, + columnNodeDefs, + recordOwnerUuid, + offset, + limit, + outputStream, + addCycle, + ...options, + }) } /** From 36b31006ef8370bff1f486acb170a88dcd2682a2 Mon Sep 17 00:00:00 2001 From: SteRiccio <1219739+SteRiccio@users.noreply.github.com> Date: Wed, 30 Jul 2025 14:09:55 +0200 Subject: [PATCH 7/7] fixing olap table query (WIP) --- common/model/db/sql/sqlSelectOlapBuilder.js | 8 +- common/model/db/tables/olapData/table.js | 12 ++ common/model/query/query/query.js | 2 +- .../modules/analysis/manager/chain/index.js | 5 + server/modules/analysis/manager/index.js | 11 +- .../surveyRdb/manager/surveyRdbManager.js | 147 +++++++++++------- .../repository/olapDataTable/index.js | 1 + .../repository/olapDataTable/read.js | 8 +- .../surveyRdb/service/surveyRdbService.js | 2 +- 9 files changed, 125 insertions(+), 71 deletions(-) diff --git a/common/model/db/sql/sqlSelectOlapBuilder.js b/common/model/db/sql/sqlSelectOlapBuilder.js index 7bf1a36d37..7d0274d216 100644 --- a/common/model/db/sql/sqlSelectOlapBuilder.js +++ b/common/model/db/sql/sqlSelectOlapBuilder.js @@ -3,10 +3,9 @@ import * as Survey from '@core/survey/survey' import SqlSelectBuilder from './sqlSelectBuilder' class SqlSelectOlapBuilder extends SqlSelectBuilder { - constructor({ table, entityDef }) { + constructor({ table }) { super() this._table = table - this._entityDef = entityDef } _selectNodeDef({ nodeDefUuid }) { @@ -23,8 +22,9 @@ class SqlSelectOlapBuilder extends SqlSelectBuilder { const { survey } = table const nodeDef = Survey.getNodeDefByUuid(nodeDefUuid)(survey) const columnName = table.getColumnNameByAttributeDef(nodeDef) - this.select(`SUM(${columnName} AS ${columnName}`) - this.select(`SUM(${columnName})/${SqlSelectOlapBuilder.areaAlias} AS ${columnName}_ha`) + const columnNameToDecimal = `CAST(${columnName} AS DECIMAL)` + this.select(`SUM(${columnNameToDecimal}) AS ${columnName}`) + this.select(`SUM(${columnNameToDecimal})/${SqlSelectOlapBuilder.areaAlias} AS ${columnName}_ha`) this.groupBy(columnName) return this } diff --git a/common/model/db/tables/olapData/table.js b/common/model/db/tables/olapData/table.js index 0b50b0c990..9b75402a85 100644 --- a/common/model/db/tables/olapData/table.js +++ b/common/model/db/tables/olapData/table.js @@ -29,6 +29,18 @@ export default class TableOlapData extends TableSurveyRdb { this._baseUnitDef = baseUnitDef } + get survey() { + return this._survey + } + + get entityDef() { + return this._entityDef + } + + get baseUnitDef() { + return this._baseUnitDef + } + getColumnNameByAttributeDef = (attributeDef) => { return NodeDef.getName(attributeDef) } diff --git a/common/model/query/query/query.js b/common/model/query/query/query.js index 7a3971d0eb..075d6269f6 100644 --- a/common/model/query/query/query.js +++ b/common/model/query/query/query.js @@ -52,7 +52,7 @@ export const isModeOlap = isMode(modes.olap) // utils export const hasSelection = (query) => !A.isEmpty(getEntityDefUuid(query)) && - (isModeAggregate(query) + (isModeAggregate(query) || isModeOlap(query) ? !A.isEmpty(getMeasures(query)) && !A.isEmpty(getDimensions(query)) : !A.isEmpty(getAttributeDefUuids(query))) diff --git a/server/modules/analysis/manager/chain/index.js b/server/modules/analysis/manager/chain/index.js index b48b3ebc9b..220b519701 100644 --- a/server/modules/analysis/manager/chain/index.js +++ b/server/modules/analysis/manager/chain/index.js @@ -46,6 +46,11 @@ export const create = async ({ user, surveyId }) => { // ====== READ export const { countChains, fetchChains, fetchChain } = ChainRepository +export const fetchChainWithSamplingDesign = async ({ surveyId }) => { + const chains = await ChainRepository.fetchChains({ surveyId }) + return chains.find(Chain.hasSamplingDesign) +} + // ====== UPDATE export const { updateChain } = ChainRepository diff --git a/server/modules/analysis/manager/index.js b/server/modules/analysis/manager/index.js index 8eb646dd3f..4770426521 100644 --- a/server/modules/analysis/manager/index.js +++ b/server/modules/analysis/manager/index.js @@ -1,4 +1,13 @@ // ====== Chain -export { create, countChains, fetchChains, fetchChain, updateChain, updateChainStatusExec, deleteChain } from './chain' +export { + create, + countChains, + fetchChains, + fetchChain, + fetchChainWithSamplingDesign, + updateChain, + updateChainStatusExec, + deleteChain, +} from './chain' export { cleanChains } from './chainsCleanManager' diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js index bf2731685b..52441fcc3f 100644 --- a/server/modules/surveyRdb/manager/surveyRdbManager.js +++ b/server/modules/surveyRdb/manager/surveyRdbManager.js @@ -9,26 +9,27 @@ import * as Record from '@core/record/record' import * as PromiseUtils from '@core/promiseUtils' import * as StringUtils from '@core/stringUtils' import { FileFormats, getExtensionByFileFormat } from '@core/fileFormats' - +import * as Chain from '@common/analysis/chain' +import { ChainStatisticalAnalysis } from '@common/analysis/chainStatisticalAnalysis' +import { ColumnNodeDef, TableDataNodeDef, ViewDataNodeDef } from '@common/model/db' +import { Query } from '@common/model/query' +import * as NodeDefTable from '@common/surveyRdb/nodeDefTable' + +import { db } from '@server/db/db' +import * as DbUtils from '@server/db/dbUtils' +import * as FlatDataWriter from '@server/utils/file/flatDataWriter' import * as FileUtils from '@server/utils/file/fileUtils' +import { StreamUtils } from '@server/utils/streamUtils' +import * as ChainManager from '@server/modules/analysis/manager' import * as RecordRepository from '@server/modules/record/repository/recordRepository' -import { db } from '../../../db/db' -import * as DbUtils from '../../../db/dbUtils' -import * as FlatDataWriter from '../../../utils/file/flatDataWriter' - -import { ColumnNodeDef, TableDataNodeDef, ViewDataNodeDef } from '../../../../common/model/db' - -import { Query } from '../../../../common/model/query' -import * as NodeDefTable from '../../../../common/surveyRdb/nodeDefTable' - import * as DataTableInsertRepository from '../repository/dataTableInsertRepository' import * as DataTableReadRepository from '../repository/dataTableReadRepository' import * as DataTableRepository from '../repository/dataTable' import * as DataViewRepository from '../repository/dataView' +import * as OlapDataRepository from '../repository/olapDataTable' import { SurveyRdbCsvExport } from './surveyRdbCsvExport' import { UniqueFileNamesGenerator } from './UniqueFileNamesGenerator' -import { StreamUtils } from '@server/utils/streamUtils' // ==== DDL @@ -120,42 +121,42 @@ export const fetchViewData = async (params, client = db) => { client ) - if (outputStream) { - const fields = columnNodeDefs - ? null // all fields will be included in the CSV file - : SurveyRdbCsvExport.getCsvExportFields({ - survey, - query, - addCycle, - includeCategoryItemsLabels, - expandCategoryItems, - includeInternalUuids, - includeDateCreated, - }) - const { transformers } = SurveyRdbCsvExport.getCsvObjectTransformer({ - survey, - query, - expandCategoryItems, - nullsToEmpty, - keepFileNamesUnique: true, - uniqueFileNamesGenerator, - }) - await DbUtils.stream({ - queryStream: result, - client, - processor: async (dbStream) => - FlatDataWriter.writeItemsStreamToStream({ - stream: dbStream, - fields, - options: { - objectTransformer: Objects.isEmpty(transformers) ? undefined : A.pipe(...transformers), - }, - outputStream, - fileFormat, - }), - }) + if (!outputStream) { + return result } - return result + const fields = columnNodeDefs + ? null // all fields will be included in the CSV file + : SurveyRdbCsvExport.getCsvExportFields({ + survey, + query, + addCycle, + includeCategoryItemsLabels, + expandCategoryItems, + includeInternalUuids, + includeDateCreated, + }) + const { transformers } = SurveyRdbCsvExport.getCsvObjectTransformer({ + survey, + query, + expandCategoryItems, + nullsToEmpty, + keepFileNamesUnique: true, + uniqueFileNamesGenerator, + }) + await DbUtils.stream({ + queryStream: result, + client, + processor: async (dbStream) => + FlatDataWriter.writeItemsStreamToStream({ + stream: dbStream, + fields, + options: { + objectTransformer: Objects.isEmpty(transformers) ? undefined : A.pipe(...transformers), + }, + outputStream, + fileFormat, + }), + }) } /** @@ -192,35 +193,61 @@ export const fetchViewDataAgg = async (params, client = db) => { client ) - if (outputStream) { - const fields = SurveyRdbCsvExport.getCsvExportFieldsAgg({ survey, query, options }) - return DbUtils.stream({ - queryStream: result, - client, - processor: async (dbStream) => - FlatDataWriter.writeItemsStreamToStream({ stream: dbStream, outputStream, fields, fileFormat }), - }) + if (!outputStream) { + return result } - return result + const fields = SurveyRdbCsvExport.getCsvExportFieldsAgg({ survey, query, options }) + return DbUtils.stream({ + queryStream: result, + client, + processor: async (dbStream) => + FlatDataWriter.writeItemsStreamToStream({ stream: dbStream, outputStream, fields, fileFormat }), + }) } -export const fetchOlapData = async (params, client = db) => { - const { survey, cycle, query, recordOwnerUuid = null, limit, offset, outputStream = null, options = {} } = params +export const fetchOlapData = async ( + { survey, cycle, query, limit, offset, outputStream = null, options = {} }, + client = db +) => { const { fileFormat = FileFormats.csv } = options - // Fetch data - const result = await DataViewRepository.fetchViewDataAgg( + const surveyId = Survey.getId(survey) + const chain = await ChainManager.fetchChainWithSamplingDesign({ surveyId }) + const baseUnitDef = chain ? Survey.getBaseUnitNodeDef({ chain })(survey) : null + const chainStatisticalAnalysis = Chain.getStatisticalAnalysis(chain) + const reportingEntityDefUuid = chainStatisticalAnalysis + ? ChainStatisticalAnalysis.getEntityDefUuid(chainStatisticalAnalysis) + : null + const reportingEntityDef = reportingEntityDefUuid ? Survey.getNodeDefByUuid(reportingEntityDefUuid)(survey) : null + + if (!chain || !reportingEntityDef) { + return null + } + // Fetch data + const result = await OlapDataRepository.fetchOlapData( { survey, cycle, query, - recordOwnerUuid, + baseUnitDef, + entityDef: reportingEntityDef, limit, offset, stream: Boolean(outputStream), }, client ) + + if (!outputStream) { + return result + } + const fields = SurveyRdbCsvExport.getCsvExportFieldsAgg({ survey, query, options }) + return DbUtils.stream({ + queryStream: result, + client, + processor: async (dbStream) => + FlatDataWriter.writeItemsStreamToStream({ stream: dbStream, outputStream, fields, fileFormat }), + }) } const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuids, search }) => { diff --git a/server/modules/surveyRdb/repository/olapDataTable/index.js b/server/modules/surveyRdb/repository/olapDataTable/index.js index 7266909356..6c294b0a03 100644 --- a/server/modules/surveyRdb/repository/olapDataTable/index.js +++ b/server/modules/surveyRdb/repository/olapDataTable/index.js @@ -1,3 +1,4 @@ export { createOlapDataTable } from './create' export { insertOlapData } from './insert' export { clearOlapData } from './delete' +export { fetchOlapData } from './read' diff --git a/server/modules/surveyRdb/repository/olapDataTable/read.js b/server/modules/surveyRdb/repository/olapDataTable/read.js index 6448609374..186fee4f73 100644 --- a/server/modules/surveyRdb/repository/olapDataTable/read.js +++ b/server/modules/surveyRdb/repository/olapDataTable/read.js @@ -8,11 +8,11 @@ import * as Survey from '@core/survey/survey' import { db } from '@server/db/db' -const _getSelectQuery = ({ survey, cycle, query }) => { +const _getSelectQuery = ({ survey, cycle, query, baseUnitDef }) => { const entityDefUuid = Query.getEntityDefUuid(query) const entityDef = Survey.getNodeDefByUuid(entityDefUuid)(survey) - const table = new TableOlapData({ survey, cycle, entityDef }) + const table = new TableOlapData({ survey, cycle, baseUnitDef, entityDef }) const queryBuilder = new SqlSelectOlapBuilder({ table, entityDef }) @@ -38,8 +38,8 @@ const _getSelectQuery = ({ survey, cycle, query }) => { return { select: queryBuilder.build(), queryParams: queryBuilder.params } } -export const selectFromOlapDataTable = async ({ survey, cycle, query, baseUnitDef, entityDef }, client = db) => { - const { select, queryParams } = _getSelectQuery({ survey, cycle, query }) +export const fetchOlapData = async ({ survey, cycle, query, baseUnitDef, entityDef }, client = db) => { + const { select, queryParams } = _getSelectQuery({ survey, cycle, query, baseUnitDef }) const table = new TableOlapData({ survey, cycle, baseUnitDef, entityDef }) const dimensionUuids = Query.getDimensions(query) diff --git a/server/modules/surveyRdb/service/surveyRdbService.js b/server/modules/surveyRdb/service/surveyRdbService.js index e710db4b33..5b9af7b8c6 100644 --- a/server/modules/surveyRdb/service/surveyRdbService.js +++ b/server/modules/surveyRdb/service/surveyRdbService.js @@ -83,7 +83,7 @@ export const fetchViewData = async (params) => { }) } if (Query.isModeOlap(parsedQuery)) { - // TODO + return SurveyRdbManager.fetchOlapData({ survey, cycle, query, limit, offset, outputStream, options }) } return SurveyRdbManager.fetchViewData({ survey,