(#24): work on tabular reporter

univie-nutrition · Jul 4, 2024 · 9ca5fa3 · 9ca5fa3
1 parent fee873c
commit 9ca5fa3
Show file tree

Hide file tree

Showing 11 changed files with 355 additions and 155 deletions.
diff --git a/globodiet/schema/src/main/resources/survey/Campaign.yaml b/globodiet/schema/src/main/resources/survey/Campaign.yaml
@@ -61,6 +61,22 @@ Campaign:
       multiLine: 4
       description: |
         Detailed information for this campaign.
+#    sidSystemId:
+#      column: SYSID
+#      column-type: nvarchar(40)
+#      required: true
+#      unique: false
+#      description: |
+#        Semantic Identifiers (SIDs) come as pair of SystemId and ObjectId.
+#        This specifies the SystemId part (e.g. when generating data exports).
+#    nutMapPath:
+#      column: NUTMAPPATH
+#      column-type: nvarchar(120)
+#      required: true
+#      unique: false
+#      description: |
+#        Specifies the named path within the configured BlobStore, that points to the QualifiedMap data used for 
+#        food-consumtion to food-component resolution (nutrient mapping).
     correction:
       column: CORRECTION
       column-type: nvarchar(64000)

diff --git a/globodiet/survey/src/main/java/dita/globodiet/survey/dom/Campaign_downloadMappingTodos.java b/globodiet/survey/src/main/java/dita/globodiet/survey/dom/Campaign_downloadMappingTodos.java
@@ -36,7 +36,7 @@
 
 import lombok.RequiredArgsConstructor;
 
-import dita.recall24.reporter.todo.TodoReportUtils;
+import dita.recall24.reporter.todo.TodoReporters;
 import io.github.causewaystuff.blobstore.applib.BlobStore;
 
 @Action(
@@ -63,7 +63,7 @@ public Clob act() {
         var systemId = "GD-AT20240507"; //TODO get from Campaign or Survey?
 
         var yaml = new StringBuilder();
-        var todoReporter = new TodoReportUtils.TodoReporter(systemId, nutMapping, interviewSet);
+        var todoReporter = new TodoReporters.TodoReporter(interviewSet, systemId, nutMapping);
         todoReporter.report(
                 DataSink.ofStringConsumer(yaml, StandardCharsets.UTF_8));
 

diff --git a/globodiet/survey/src/main/java/dita/globodiet/survey/dom/Campaign_generateReport.java b/globodiet/survey/src/main/java/dita/globodiet/survey/dom/Campaign_generateReport.java
@@ -34,8 +34,8 @@
 
 import lombok.RequiredArgsConstructor;
 
-import dita.recall24.reporter.tabular.TabularReportUtil;
-import dita.recall24.reporter.tabular.TabularReportUtil.Aggregation;
+import dita.recall24.reporter.tabular.TabularReporters;
+import dita.recall24.reporter.tabular.TabularReporters.Aggregation;
 import io.github.causewaystuff.blobstore.applib.BlobStore;
 
 @Action(
@@ -60,9 +60,11 @@ public Clob act(@Parameter final Aggregation aggregation) {
         var interviewSet = Campaigns.interviewSet(mixee, surveyBlobStore);
 
         //TODO flesh out reporting
-        var tabularReport = new TabularReportUtil.TabularReport(interviewSet, Aggregation.NONE);
+        var tabularReport = new TabularReporters.TabularReport(interviewSet, null, null, null, null, null, null, aggregation);
 
-        return Clob.of("report", CommonMimeType.YAML, interviewSet.toYaml());
+        var name = String.format("report-%s", aggregation.name());
+
+        return Clob.of(name, CommonMimeType.YAML, interviewSet.toYaml());
     }
 
 }
diff --git a/globodiet/survey/src/main/java/dita/globodiet/survey/dom/Campaigns.java b/globodiet/survey/src/main/java/dita/globodiet/survey/dom/Campaigns.java
@@ -27,6 +27,7 @@
 
 import lombok.experimental.UtilityClass;
 
+import dita.commons.food.composition.FoodCompositionRepository;
 import dita.commons.qmap.QualifiedMap;
 import dita.commons.types.Message;
 import dita.globodiet.survey.util.InterviewUtils;
@@ -45,7 +46,9 @@ public class Campaigns {
     enum DataSourceLocation {
         INTERVIEW,
         FCDB,
-        QMAP;
+        QMAP_NUT,
+        QMAP_FCO,
+        QMAP_POC;
         NamedPath namedPath(final Campaign campaign) {
             if(campaign==null
                     || _Strings.isNullOrEmpty(campaign.getSurveyCode())
@@ -56,7 +59,9 @@ NamedPath namedPath(final Campaign campaign) {
             return switch(this) {
                 case INTERVIEW -> root.add("campaigns").add(NamedPath.of(campaign.getCode().toLowerCase()));
                 case FCDB -> root.add("fcdb").add("fcdb.yaml.7z");
-                case QMAP -> root.add("qmap").add("qmap.yaml.7z");
+                case QMAP_NUT -> root.add("qmap").add("nut.yaml.7z");
+                case QMAP_FCO -> root.add("qmap").add("fco.yaml");
+                case QMAP_POC -> root.add("qmap").add("poc.yaml");
             };
         }
     }
@@ -87,22 +92,38 @@ public InterviewSet24.Dto interviewSet(
         return interviewSet;
     }
 
-    // -- NUT MAPPING (Q-MAP)
+    // -- FCDB
 
-    public QualifiedMap nutMapping(
+    public FoodCompositionRepository fcdb(
             final Campaign campaign,
             final BlobStore blobStore) {
 
-        var mapDataSource = SevenZUtils.decompress(
-                blobStore
-                    .lookupBlob(DataSourceLocation.QMAP.namedPath(campaign))
-                    .orElseThrow()
-                    .asDataSource());
+        var fcdbDataSource = blobStore.lookupBlob(DataSourceLocation.FCDB.namedPath(campaign))
+                .orElseThrow()
+                .asDataSource();
+        var foodCompositionRepo = FoodCompositionRepository.tryFromYaml(SevenZUtils.decompress(fcdbDataSource))
+                .valueAsNonNullElseFail();
+        return foodCompositionRepo;
+    }
+
+    // -- Q-MAP
 
-        var qMap = QualifiedMap.tryFromYaml(mapDataSource)
-            .valueAsNonNullElseFail();
+    public QualifiedMap nutMapping(
+            final Campaign campaign,
+            final BlobStore blobStore) {
+        return loadQmap(DataSourceLocation.QMAP_NUT, campaign, blobStore);
+    }
+
+    public QualifiedMap fcoMapping(
+            final Campaign campaign,
+            final BlobStore blobStore) {
+        return loadQmap(DataSourceLocation.QMAP_FCO, campaign, blobStore);
+    }
 
-        return qMap;
+    public QualifiedMap pocMapping(
+            final Campaign campaign,
+            final BlobStore blobStore) {
+        return loadQmap(DataSourceLocation.QMAP_POC, campaign, blobStore);
     }
 
     // -- HELPER
@@ -137,4 +158,21 @@ private final RecallNode24.Annotation toAnnotation() {
 
     }
 
+    private QualifiedMap loadQmap(
+            final DataSourceLocation loc,
+            final Campaign campaign,
+            final BlobStore blobStore) {
+        var mapDataSource =
+                blobStore
+                    .lookupBlob(loc.namedPath(campaign))
+                    .orElseThrow()
+                    .asDataSource();
+        switch(loc) {
+            case QMAP_NUT ->
+                mapDataSource = SevenZUtils.decompress(mapDataSource);
+            default -> {}
+        }
+        return QualifiedMap.tryFromYaml(mapDataSource).valueAsNonNullElseFail();
+    }
+
 }
diff --git a/globodiet/survey/src/test/java/dita/globodiet/survey/DitaGdSurveyIntegrationTest.java b/globodiet/survey/src/test/java/dita/globodiet/survey/DitaGdSurveyIntegrationTest.java
@@ -18,66 +18,50 @@
  */
 package dita.globodiet.survey;
 
-import java.util.function.Consumer;
-import java.util.stream.Stream;
-
 import jakarta.inject.Inject;
 
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.lang.Nullable;
 
 import org.apache.causeway.testing.integtestsupport.applib.CausewayIntegrationTestAbstract;
 
-import lombok.NonNull;
-
 import dita.commons.food.composition.FoodCompositionRepository;
 import dita.commons.qmap.QualifiedMap;
-import dita.commons.types.Message;
-import dita.globodiet.survey.recall24.InterviewXmlParser;
-import dita.globodiet.survey.util.InterviewUtils;
-import dita.recall24.dto.Correction24;
+import dita.globodiet.survey.dom.Campaign;
+import dita.globodiet.survey.dom.Campaigns;
 import dita.recall24.dto.InterviewSet24;
-import dita.recall24.dto.util.Recall24DtoUtils;
 import io.github.causewaystuff.blobstore.applib.BlobStore;
-import io.github.causewaystuff.commons.base.types.NamedPath;
-import io.github.causewaystuff.commons.compression.SevenZUtils;
 
 public abstract class DitaGdSurveyIntegrationTest
 extends CausewayIntegrationTestAbstract {
 
     @Inject @Qualifier("survey") protected BlobStore surveyBlobStore;
 
     protected FoodCompositionRepository loadFcdb() {
-        var fcdbDataSource = SevenZUtils.decompress(
-                surveyBlobStore.lookupBlob(NamedPath.of("fcdb", "fcdb.yaml.7z")).orElseThrow().asDataSource());
-
-        var foodCompositionRepo = FoodCompositionRepository.tryFromYaml(fcdbDataSource)
-            .valueAsNonNullElseFail();
-
-        return foodCompositionRepo;
+        return Campaigns.fcdb(campaignForTesting(), surveyBlobStore);
     }
 
     protected QualifiedMap loadNutMapping() {
-        var mapDataSource = SevenZUtils.decompress(
-                surveyBlobStore.lookupBlob(NamedPath.of("qmap", "qmap.yaml.7z")).orElseThrow().asDataSource());
+        return Campaigns.nutMapping(campaignForTesting(), surveyBlobStore);
+    }
 
-        var qMap = QualifiedMap.tryFromYaml(mapDataSource)
-            .valueAsNonNullElseFail();
+    protected QualifiedMap loadFcoMapping() {
+        return Campaigns.fcoMapping(campaignForTesting(), surveyBlobStore);
+    }
 
-        return qMap;
+    protected QualifiedMap loadPocMapping() {
+        return Campaigns.pocMapping(campaignForTesting(), surveyBlobStore);
     }
 
-    protected Stream<InterviewSet24.Dto> loadAndStreamInterviews(
-            final @NonNull NamedPath path,
-            final @Nullable Correction24 correction,
-            final @Nullable Consumer<Message> messageConsumer) {
-        return InterviewUtils.streamSources(surveyBlobStore, path, true)
-            .map(ds->InterviewXmlParser.parse(ds, messageConsumer))
-            .map(Recall24DtoUtils.correct(correction));
+    protected InterviewSet24.Dto loadInterviewSet() {
+        return Campaigns.interviewSet(campaignForTesting(), surveyBlobStore);
     }
 
-    protected Correction24 loadCorrection() {
-        return Correction24.tryFromYaml("""
+    // -- HELPER
+
+    private Campaign campaignForTesting() {
+        var campaign = new Campaign();
+        campaign.setCode("wave1");
+        campaign.setCorrection("""
                 respondents:
                 - alias: "EB0070"
                   newAlias: "EB_0070"
@@ -91,8 +75,15 @@ protected Correction24 loadCorrection() {
                   dateOfBirth: "2002-09-21"
                 - alias: "EB_0093"
                   sex: FEMALE
-                """)
-                .valueAsNullableElseFail();
+                - alias: "EB_0088"
+                  dateOfBirth: "1967-06-09"
+                - alias: "EB_0032"
+                  dateOfBirth: "1999-03-04"
+                - alias: "EB_0116"
+                  dateOfBirth: "1998-01-26"
+                """);
+        campaign.setSurveyCode("at-national-2026");
+        return campaign;
     }
 
 }
diff --git a/...urvey/src/test/java/dita/globodiet/survey/recall24/InterviewXmlParserIntegrationTest.java b/...urvey/src/test/java/dita/globodiet/survey/recall24/InterviewXmlParserIntegrationTest.java
@@ -19,30 +19,32 @@
 package dita.globodiet.survey.recall24;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.List;
 import java.util.function.Consumer;
 
 import org.junit.jupiter.api.Test;
 
 import org.springframework.boot.test.context.SpringBootTest;
 
-import org.apache.causeway.applib.graph.tree.TreeNode;
 import org.apache.causeway.commons.collections.Can;
-import org.apache.causeway.commons.io.DataSink;
+
+import lombok.val;
 
 import dita.commons.qmap.QualifiedMap;
 import dita.commons.qmap.QualifiedMap.QualifiedMapKey;
 import dita.commons.qmap.QualifiedMapEntry;
+import dita.commons.sid.SemanticIdentifier;
+import dita.commons.sid.SemanticIdentifierSet;
 import dita.globodiet.survey.DitaGdSurveyIntegrationTest;
 import dita.globodiet.survey.DitaTestModuleGdSurvey;
 import dita.globodiet.survey.PrivateDataTest;
-import dita.globodiet.survey.util.InterviewUtils;
 import dita.recall24.dto.RecallNode24;
 import dita.recall24.dto.Record24;
-import dita.recall24.dto.util.Recall24DtoUtils;
 import dita.recall24.dto.util.Recall24SummaryStatistics;
 import dita.recall24.dto.util.Recall24SummaryStatistics.MappingTodo;
-import dita.recall24.reporter.todo.TodoReportUtils;
-import io.github.causewaystuff.commons.base.types.NamedPath;
+import dita.recall24.reporter.tabular.TabularReporters;
+import dita.recall24.reporter.tabular.TabularReporters.Aggregation;
 
 @SpringBootTest(classes = {
         DitaTestModuleGdSurvey.class,
@@ -56,28 +58,46 @@ void parsingFromBlobStore() {
         final var systemId = "GD-AT20240507";
 
         var nutMapping = loadNutMapping();
-        var correction = loadCorrection();
+        var fcoMapping = loadFcoMapping();
+        var pocMapping = loadPocMapping();
+        var interviewSet = loadInterviewSet();
 
         var stats = new Recall24SummaryStatistics();
         var recordProcessor = new RecordProcessor(stats, systemId, nutMapping);
 
-        var interviewSet = InterviewUtils
-                .interviewSetFromBlobStrore(NamedPath.of("at-national-2026"), surveyBlobStore, correction, null)
-                //.transform(new NutriDbConverters.ToNutriDbTransfomer())
-                ;
+        //TODO flesh out reporting
+        var xlsxFile = new File("d:/tmp/_scratch/report-no-aggregates.xlsx");
+        var tabularReport = new TabularReporters.TabularReport(interviewSet, systemId,
+                nutMapping,
+                fcoMapping, SemanticIdentifierSet.ofCollection(List.of(new SemanticIdentifier("Language", "de"))),
+                pocMapping, SemanticIdentifierSet.ofCollection(List.of(new SemanticIdentifier("Language", "de"))),
+                Aggregation.NONE);
+        tabularReport.report(xlsxFile);
+
+        val pb = new ProcessBuilder();
+
+        pb.directory(new File("d:/tmp/_scratch"));
+        pb.command(List.of(
+                "C:/Program Files/LibreOffice/program/scalc.exe",
+                xlsxFile.getAbsolutePath()));
+        pb.inheritIO();
+
+        try {
+            pb.start().waitFor();
+        } catch (InterruptedException | IOException e) {
+            e.printStackTrace();
+        }
 
-        var todoReporter = new TodoReportUtils.TodoReporter(systemId, nutMapping, interviewSet);
-        todoReporter.report(
-                DataSink.ofFile(new File("d:/tmp/_scratch/mapping-todos.txt")));
+//        var todoReporter = new TodoReportUtils.TodoReporter(systemId, nutMapping, interviewSet);
+//        todoReporter.report(
+//                DataSink.ofFile(new File("d:/tmp/_scratch/mapping-todos.txt")));
 
-        Recall24DtoUtils.wrapAsTreeNode(interviewSet)
-            .streamDepthFirst()
-            .map(TreeNode::getValue)
-            .forEach((RecallNode24 node)->{
-                stats.accept((dita.recall24.dto.RecallNode24) node);
+        interviewSet.streamDepthFirst()
+            .forEach((final RecallNode24 node)->{
+                stats.accept(node);
                 switch(node) {
-                case Record24.Consumption cRec -> recordProcessor.accept(cRec);
-                default -> {}
+                    case Record24.Consumption cRec -> recordProcessor.accept(cRec);
+                    default -> {}
                 }
             });
 

diff --git a/recall24/dto/src/main/java/dita/recall24/dto/InterviewSet24.java b/recall24/dto/src/main/java/dita/recall24/dto/InterviewSet24.java
@@ -31,6 +31,7 @@
 
 import org.springframework.lang.Nullable;
 
+import org.apache.causeway.applib.graph.tree.TreeNode;
 import org.apache.causeway.commons.collections.Can;
 import org.apache.causeway.commons.internal.base._NullSafe;
 import org.apache.causeway.commons.io.JsonUtils;
@@ -132,6 +133,15 @@ public Stream<Interview24.Dto> streamInterviews() {
                     .flatMap(resp->resp.interviews().stream());
         }
 
+        /**
+         * @implNote requires Causewaystuff tree metamodel integration
+         */
+        public Stream<RecallNode24> streamDepthFirst() {
+            return Recall24DtoUtils.wrapAsTreeNode(this)
+                .streamDepthFirst()
+                .map(TreeNode::getValue);
+        }
+
         /**
          * Returns a new tree with the transformed nodes.
          * @param transformer - transforms fields only (leave parent child relations untouched)