OLAP: query

common/model/db/sql/sqlSelectOlapBuilder.js

@@ -0,0 +1,39 @@
+import * as Survey from '@core/survey/survey'
+
+import SqlSelectBuilder from './sqlSelectBuilder'
+
+class SqlSelectOlapBuilder extends SqlSelectBuilder {
+  constructor({ table }) {
+    super()
+    this._table = table
+  }
+
+  _selectNodeDef({ nodeDefUuid }) {
+    const { _table: table } = this
+    const { survey } = table
+    const nodeDef = Survey.getNodeDefByUuid(nodeDefUuid)(survey)
+    const columnName = table.getColumnNameByAttributeDef(nodeDef)
+    this.select(columnName)
+    return this
+  }
+
+  selectMeasure({ nodeDefUuid }) {
+    const { _table: table } = this
+    const { survey } = table
+    const nodeDef = Survey.getNodeDefByUuid(nodeDefUuid)(survey)
+    const columnName = table.getColumnNameByAttributeDef(nodeDef)
+    const columnNameToDecimal = `CAST(${columnName} AS DECIMAL)`
+    this.select(`SUM(${columnNameToDecimal}) AS ${columnName}`)
+    this.select(`SUM(${columnNameToDecimal})/${SqlSelectOlapBuilder.areaAlias} AS ${columnName}_ha`)
+    this.groupBy(columnName)
+    return this
+  }
+
+  selectDimension({ nodeDefUuid }) {
+    return this._selectNodeDef({ nodeDefUuid })
+  }
+}
+
+SqlSelectOlapBuilder.areaAlias = 'area'
+
+export default SqlSelectOlapBuilder

common/model/db/tables/olapData/olapAreaView.js

@@ -0,0 +1,25 @@
+import * as Survey from '@core/survey/survey'
+import * as NodeDef from '@core/survey/nodeDef'
+
+import TableSurveyRdb from '../tableSurveyRdb'
+import TableOlapData from './table'
+
+const generateViewName = ({ survey, cycle, entityDef, baseUnitDef }) => {
+  const dataTable = new TableOlapData({ survey, cycle, entityDef, baseUnitDef })
+  return dataTable.name + '_area_view'
+}
+
+export default class OlapAreaView extends TableSurveyRdb {
+  constructor({ survey, cycle, entityDef, baseUnitDef }) {
+    super(Survey.getId(survey), generateViewName({ survey, cycle, entityDef, baseUnitDef }))
+    this._baseUnitDef = baseUnitDef
+  }
+
+  get baseUnitUuidColumnName() {
+    return NodeDef.getName(this._baseUnitDef) + '_uuid'
+  }
+
+  get areaColumnName() {
+    return 'area'
+  }
+}

common/model/db/tables/olapData/table.js

@@ -29,6 +29,22 @@ export default class TableOlapData extends TableSurveyRdb {
     this._baseUnitDef = baseUnitDef
   }
 
+  get survey() {
+    return this._survey
+  }
+
+  get entityDef() {
+    return this._entityDef
+  }
+
+  get baseUnitDef() {
+    return this._baseUnitDef
+  }
+
+  getColumnNameByAttributeDef = (attributeDef) => {
+    return NodeDef.getName(attributeDef)
+  }
+
   get attributeDefsForColumns() {
     const sortedAttributeDefs = []
     const attributeDefsByUuid = {}
@@ -64,7 +80,7 @@ export default class TableOlapData extends TableSurveyRdb {
       // base unit UUID
       this.baseUnitUuidColumnName,
       // attribute columns
-      ...this.attributeDefsForColumns.map(NodeDef.getName),
+      ...this.attributeDefsForColumns.map((attributeDef) => this.getColumnNameByAttributeDef(attributeDef)),
     ]
   }
 
@@ -87,6 +103,10 @@ export default class TableOlapData extends TableSurveyRdb {
     return NodeDef.getName(this._baseUnitDef) + '_uuid'
   }
 
+  get expFactorColumnName() {
+    return baseColumnSet.expFactor
+  }
+
   get columnNamesAndTypes() {
     return [
       // base columns

common/model/query/query/keys.js

@@ -15,6 +15,7 @@ export const modes = {
   raw: 'raw',
   rawEdit: 'rawEdit',
   aggregate: 'aggregate',
+  olap: 'olap',
 }
 
 export const displayTypes = {

common/model/query/query/query.js

@@ -47,11 +47,12 @@ const isMode = (mode) => (query) => getMode(query) === mode
 export const isModeAggregate = isMode(modes.aggregate)
 export const isModeRaw = isMode(modes.raw)
 export const isModeRawEdit = isMode(modes.rawEdit)
+export const isModeOlap = isMode(modes.olap)
 
 // utils
 export const hasSelection = (query) =>
   !A.isEmpty(getEntityDefUuid(query)) &&
-  (isModeAggregate(query)
+  (isModeAggregate(query) || isModeOlap(query)
     ? !A.isEmpty(getMeasures(query)) && !A.isEmpty(getDimensions(query))
     : !A.isEmpty(getAttributeDefUuids(query)))
 

core/i18n/resources/en/common.js

@@ -749,6 +749,7 @@ Please refine your query (e.g. adding a filter) to reduce the number of items.
       mode: {
         label: 'Mode:',
         aggregate: 'Aggregate',
+        olap: 'OLAP',
         raw: 'Raw',
         rawEdit: 'Raw edit',
       },

core/survey/nodeDef.js

@@ -312,7 +312,13 @@ export const getMeta = R.propOr({}, keys.meta)
 export const getMetaHierarchy = R.pathOr([], [keys.meta, metaKeys.h])
 
 // Utils
-export const getLabel = (nodeDef, lang, type = NodeDefLabelTypes.label, defaultToName = true) => {
+const getLabelSuffix = (nodeDef) => {
+  if (isVirtual(nodeDef)) return ' (V)'
+  if (isAnalysis(nodeDef) && isAttribute(nodeDef)) return ' (C)'
+  return ''
+}
+
+export const getLabel = (nodeDef, lang, type = NodeDefLabelTypes.label, defaultToName = true, includeSuffix = true) => {
   let firstPart = ''
   const name = getName(nodeDef)
 
@@ -333,12 +339,10 @@ export const getLabel = (nodeDef, lang, type = NodeDefLabelTypes.label, defaultT
   if (StringUtils.isBlank(firstPart) && defaultToName) {
     firstPart = name
   }
-
-  const suffix = isVirtual(nodeDef) ? ' (V)' : isAnalysis(nodeDef) && isAttribute(nodeDef) ? ' (C)' : ''
-
-  return firstPart + suffix
+  return firstPart + (includeSuffix ? getLabelSuffix(nodeDef) : '')
 }
-export const getLabelWithType = ({ nodeDef, lang, type }) => getLabel(nodeDef, lang, type)
+export const getLabelWithType = ({ nodeDef, lang, type, includeSuffix }) =>
+  getLabel(nodeDef, lang, type, true, includeSuffix)
 
 export const getDescription = (lang) => (nodeDef) => R.propOr('', lang, getDescriptions(nodeDef))
 

server/modules/analysis/manager/chain/index.js

@@ -46,6 +46,11 @@ export const create = async ({ user, surveyId }) => {
 // ====== READ
 export const { countChains, fetchChains, fetchChain } = ChainRepository
 
+export const fetchChainWithSamplingDesign = async ({ surveyId }) => {
+  const chains = await ChainRepository.fetchChains({ surveyId })
+  return chains.find(Chain.hasSamplingDesign)
+}
+
 // ====== UPDATE
 export const { updateChain } = ChainRepository
 

server/modules/analysis/manager/index.js

@@ -1,4 +1,13 @@
 // ====== Chain
-export { create, countChains, fetchChains, fetchChain, updateChain, updateChainStatusExec, deleteChain } from './chain'
+export {
+  create,
+  countChains,
+  fetchChains,
+  fetchChain,
+  fetchChainWithSamplingDesign,
+  updateChain,
+  updateChainStatusExec,
+  deleteChain,
+} from './chain'
 
 export { cleanChains } from './chainsCleanManager'

server/modules/surveyRdb/manager/surveyRdbManager.js

@@ -9,26 +9,27 @@ import * as Record from '@core/record/record'
 import * as PromiseUtils from '@core/promiseUtils'
 import * as StringUtils from '@core/stringUtils'
 import { FileFormats, getExtensionByFileFormat } from '@core/fileFormats'
-
+import * as Chain from '@common/analysis/chain'
+import { ChainStatisticalAnalysis } from '@common/analysis/chainStatisticalAnalysis'
+import { ColumnNodeDef, TableDataNodeDef, ViewDataNodeDef } from '@common/model/db'
+import { Query } from '@common/model/query'
+import * as NodeDefTable from '@common/surveyRdb/nodeDefTable'
+
+import { db } from '@server/db/db'
+import * as DbUtils from '@server/db/dbUtils'
+import * as FlatDataWriter from '@server/utils/file/flatDataWriter'
 import * as FileUtils from '@server/utils/file/fileUtils'
+import { StreamUtils } from '@server/utils/streamUtils'
+import * as ChainManager from '@server/modules/analysis/manager'
 import * as RecordRepository from '@server/modules/record/repository/recordRepository'
 
-import { db } from '../../../db/db'
-import * as DbUtils from '../../../db/dbUtils'
-import * as FlatDataWriter from '../../../utils/file/flatDataWriter'
-
-import { ColumnNodeDef, TableDataNodeDef, ViewDataNodeDef } from '../../../../common/model/db'
-
-import { Query } from '../../../../common/model/query'
-import * as NodeDefTable from '../../../../common/surveyRdb/nodeDefTable'
-
 import * as DataTableInsertRepository from '../repository/dataTableInsertRepository'
 import * as DataTableReadRepository from '../repository/dataTableReadRepository'
 import * as DataTableRepository from '../repository/dataTable'
 import * as DataViewRepository from '../repository/dataView'
+import * as OlapDataRepository from '../repository/olapDataTable'
 import { SurveyRdbCsvExport } from './surveyRdbCsvExport'
 import { UniqueFileNamesGenerator } from './UniqueFileNamesGenerator'
-import { StreamUtils } from '@server/utils/streamUtils'
 
 // ==== DDL
 
@@ -55,6 +56,7 @@ export { createNodeKeysHierarchyView } from '../repository/nodeKeysHierarchyView
 export { deleteNodeResultsByChainUuid, MassiveUpdateData, MassiveUpdateNodes } from '../repository/resultNode'
 
 export { createOlapDataTable, insertOlapData, clearOlapData } from '../repository/olapDataTable'
+export { createOlapAreaView } from '../repository/olapAreaView'
 
 const maxExcelCellsLimit = 1000000
 
@@ -119,42 +121,42 @@ export const fetchViewData = async (params, client = db) => {
     client
   )
 
-  if (outputStream) {
-    const fields = columnNodeDefs
-      ? null // all fields will be included in the CSV file
-      : SurveyRdbCsvExport.getCsvExportFields({
-          survey,
-          query,
-          addCycle,
-          includeCategoryItemsLabels,
-          expandCategoryItems,
-          includeInternalUuids,
-          includeDateCreated,
-        })
-    const { transformers } = SurveyRdbCsvExport.getCsvObjectTransformer({
-      survey,
-      query,
-      expandCategoryItems,
-      nullsToEmpty,
-      keepFileNamesUnique: true,
-      uniqueFileNamesGenerator,
-    })
-    await DbUtils.stream({
-      queryStream: result,
-      client,
-      processor: async (dbStream) =>
-        FlatDataWriter.writeItemsStreamToStream({
-          stream: dbStream,
-          fields,
-          options: {
-            objectTransformer: Objects.isEmpty(transformers) ? undefined : A.pipe(...transformers),
-          },
-          outputStream,
-          fileFormat,
-        }),
-    })
+  if (!outputStream) {
+    return result
   }
-  return result
+  const fields = columnNodeDefs
+    ? null // all fields will be included in the CSV file
+    : SurveyRdbCsvExport.getCsvExportFields({
+        survey,
+        query,
+        addCycle,
+        includeCategoryItemsLabels,
+        expandCategoryItems,
+        includeInternalUuids,
+        includeDateCreated,
+      })
+  const { transformers } = SurveyRdbCsvExport.getCsvObjectTransformer({
+    survey,
+    query,
+    expandCategoryItems,
+    nullsToEmpty,
+    keepFileNamesUnique: true,
+    uniqueFileNamesGenerator,
+  })
+  await DbUtils.stream({
+    queryStream: result,
+    client,
+    processor: async (dbStream) =>
+      FlatDataWriter.writeItemsStreamToStream({
+        stream: dbStream,
+        fields,
+        options: {
+          objectTransformer: Objects.isEmpty(transformers) ? undefined : A.pipe(...transformers),
+        },
+        outputStream,
+        fileFormat,
+      }),
+  })
 }
 
 /**
@@ -191,16 +193,61 @@ export const fetchViewDataAgg = async (params, client = db) => {
     client
   )
 
-  if (outputStream) {
-    const fields = SurveyRdbCsvExport.getCsvExportFieldsAgg({ survey, query, options })
-    return DbUtils.stream({
-      queryStream: result,
-      client,
-      processor: async (dbStream) =>
-        FlatDataWriter.writeItemsStreamToStream({ stream: dbStream, outputStream, fields, fileFormat }),
-    })
+  if (!outputStream) {
+    return result
   }
-  return result
+  const fields = SurveyRdbCsvExport.getCsvExportFieldsAgg({ survey, query, options })
+  return DbUtils.stream({
+    queryStream: result,
+    client,
+    processor: async (dbStream) =>
+      FlatDataWriter.writeItemsStreamToStream({ stream: dbStream, outputStream, fields, fileFormat }),
+  })
+}
+
+export const fetchOlapData = async (
+  { survey, cycle, query, limit, offset, outputStream = null, options = {} },
+  client = db
+) => {
+  const { fileFormat = FileFormats.csv } = options
+
+  const surveyId = Survey.getId(survey)
+  const chain = await ChainManager.fetchChainWithSamplingDesign({ surveyId })
+  const baseUnitDef = chain ? Survey.getBaseUnitNodeDef({ chain })(survey) : null
+  const chainStatisticalAnalysis = Chain.getStatisticalAnalysis(chain)
+  const reportingEntityDefUuid = chainStatisticalAnalysis
+    ? ChainStatisticalAnalysis.getEntityDefUuid(chainStatisticalAnalysis)
+    : null
+  const reportingEntityDef = reportingEntityDefUuid ? Survey.getNodeDefByUuid(reportingEntityDefUuid)(survey) : null
+
+  if (!chain || !reportingEntityDef) {
+    return null
+  }
+  // Fetch data
+  const result = await OlapDataRepository.fetchOlapData(
+    {
+      survey,
+      cycle,
+      query,
+      baseUnitDef,
+      entityDef: reportingEntityDef,
+      limit,
+      offset,
+      stream: Boolean(outputStream),
+    },
+    client
+  )
+
+  if (!outputStream) {
+    return result
+  }
+  const fields = SurveyRdbCsvExport.getCsvExportFieldsAgg({ survey, query, options })
+  return DbUtils.stream({
+    queryStream: result,
+    client,
+    processor: async (dbStream) =>
+      FlatDataWriter.writeItemsStreamToStream({ stream: dbStream, outputStream, fields, fileFormat }),
+  })
 }
 
 const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuids, search }) => {