From 3b284295d7673b56fa22fab176119e95b34cb8e2 Mon Sep 17 00:00:00 2001
From: Tom Szendrey <T.Szendrey@hotmail.com>
Date: Wed, 9 Oct 2024 09:41:53 -0400
Subject: [PATCH 1/6] Adding Output Console to drilldowns with notebooks
 (#5087)

---
 .../drilldown/tera-notebook-output.vue        | 55 +++++++++++++++++++
 .../widgets/tera-resizable-panel.vue          | 25 ++++++++-
 .../model-edit/tera-model-edit-drilldown.vue  | 20 ++++---
 3 files changed, 90 insertions(+), 10 deletions(-)
 create mode 100644 packages/client/hmi-client/src/components/drilldown/tera-notebook-output.vue

diff --git a/packages/client/hmi-client/src/components/drilldown/tera-notebook-output.vue b/packages/client/hmi-client/src/components/drilldown/tera-notebook-output.vue
new file mode 100644
index 0000000000..8e31d36166
--- /dev/null
+++ b/packages/client/hmi-client/src/components/drilldown/tera-notebook-output.vue
@@ -0,0 +1,55 @@
+<template>
+	<tera-resizable-panel v-if="!isHidden" :start-height="100" :resize-from-top="true" class="container">
+		<h6>
+			<span>{{ name }}</span>
+			<Button rounded text icon="pi pi-times" @click="isHidden = true" />
+		</h6>
+		<code class="code-section">{{ props.traceback }}</code>
+	</tera-resizable-panel>
+	<div v-if="isHidden" class="container">
+		<h6>
+			<span>{{ name }}</span>
+			<Button rounded text icon="pi pi-angle-double-up" @click="isHidden = false" />
+		</h6>
+	</div>
+</template>
+
+<script setup lang="ts">
+import { ref } from 'vue';
+import Button from 'primevue/button';
+import TeraResizablePanel from '@/components/widgets/tera-resizable-panel.vue';
+
+const props = defineProps<{
+	name?: string;
+	traceback?: string;
+}>();
+
+const name = props.name ?? 'Output Console';
+const isHidden = ref<boolean>(false);
+</script>
+
+<style scoped>
+.container {
+	background-color: var(--surface-100);
+	padding: var(--gap-small);
+	border-radius: var(--border-radius);
+	border: 1px solid var(--surface-border-light);
+	/* Shadow/medium */
+	box-shadow:
+		0 2px 4px -1px rgba(0, 0, 0, 0.06),
+		0 4px 6px -1px rgba(0, 0, 0, 0.08);
+}
+
+.code-section {
+	white-space: pre-wrap;
+}
+
+h6 {
+	display: flex;
+	justify-content: space-between;
+}
+
+.hide {
+	display: none;
+}
+</style>
diff --git a/packages/client/hmi-client/src/components/widgets/tera-resizable-panel.vue b/packages/client/hmi-client/src/components/widgets/tera-resizable-panel.vue
index 5979b72f61..ab606c39a2 100644
--- a/packages/client/hmi-client/src/components/widgets/tera-resizable-panel.vue
+++ b/packages/client/hmi-client/src/components/widgets/tera-resizable-panel.vue
@@ -3,7 +3,8 @@
 		<main class="content">
 			<slot />
 		</main>
-		<div class="resize-handle" @mousedown="startResize"></div>
+		<div v-if="resizeFromTop === true" class="resize-handle-top" @mousedown="startResize"></div>
+		<div v-else class="resize-handle-bottom" @mousedown="startResize"></div>
 	</section>
 </template>
 
@@ -19,7 +20,11 @@ const containerHeight = ref(320);
 const draggedY = ref(0);
 
 const resize = (event: MouseEvent) => {
-	containerHeight.value += event.clientY - draggedY.value;
+	if (props?.resizeFromTop) {
+		containerHeight.value += -event.clientY + draggedY.value;
+	} else {
+		containerHeight.value += event.clientY - draggedY.value;
+	}
 	draggedY.value = event.clientY;
 };
 
@@ -35,6 +40,7 @@ const startResize = (event: MouseEvent) => {
 };
 const props = defineProps<{
 	startHeight?: number;
+	resizeFromTop?: boolean;
 }>();
 
 onMounted(() => {
@@ -71,7 +77,7 @@ main {
 	height: 100%;
 }
 
-.resize-handle {
+.resize-handle-bottom {
 	position: absolute;
 	bottom: 0;
 	left: 0;
@@ -84,6 +90,19 @@ main {
 	mix-blend-mode: darken;
 }
 
+.resize-handle-top {
+	position: absolute;
+	top: 0;
+	left: 0;
+	width: 100%;
+	height: 6px;
+	cursor: ns-resize;
+	background: var(--surface-border-light);
+	z-index: 1;
+	border-radius: 0 0 var(--border-radius-big) var(--border-radius-big);
+	mix-blend-mode: darken;
+}
+
 .resize-handle:hover {
 	background: var(--primary-color-light);
 }
diff --git a/packages/client/hmi-client/src/components/workflow/ops/model-edit/tera-model-edit-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/model-edit/tera-model-edit-drilldown.vue
index 248a75cbd7..c6136572c4 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/model-edit/tera-model-edit-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/model-edit/tera-model-edit-drilldown.vue
@@ -41,6 +41,7 @@
 				class="ace-editor"
 				:options="{ showPrintMargin: false }"
 			/>
+			<tera-notebook-output :traceback="executeResponseTraceback" />
 		</tera-drilldown-section>
 		<template #preview v-if="drilldownRef?.selectedTab === DrilldownTabs.Notebook">
 			<tera-drilldown-preview
@@ -51,10 +52,10 @@
 				is-selectable
 			>
 				<tera-notebook-error
-					v-if="executeResponse.status === OperatorStatus.ERROR"
-					:name="executeResponse.name"
-					:value="executeResponse.value"
-					:traceback="executeResponse.traceback"
+					v-if="executeErrorResponse.status === OperatorStatus.ERROR"
+					:name="executeErrorResponse.name"
+					:value="executeErrorResponse.value"
+					:traceback="executeErrorResponse.traceback"
 				/>
 				<tera-model v-else-if="amr" is-workflow is-save-for-reuse :assetId="amr.id" @on-save="updateNode" />
 				<tera-progress-spinner v-else-if="isUpdatingModel || !amr" is-centered :font-size="2">
@@ -88,6 +89,7 @@ import TeraDrilldownPreview from '@/components/drilldown/tera-drilldown-preview.
 import TeraDrilldownSection from '@/components/drilldown/tera-drilldown-section.vue';
 import TeraModel from '@/components/model/tera-model.vue';
 import TeraNotebookError from '@/components/drilldown/tera-notebook-error.vue';
+import TeraNotebookOutput from '@/components/drilldown/tera-notebook-output.vue';
 import TeraNotebookJupyterInput from '@/components/llm/tera-notebook-jupyter-input.vue';
 import TeraProgressSpinner from '@/components/widgets/tera-progress-spinner.vue';
 import { nodeOutputLabel } from '@/components/workflow/util';
@@ -147,13 +149,15 @@ const llmThoughts = ref<any[]>([]);
 
 const isDraft = ref(false);
 
-const executeResponse = ref({
+const executeErrorResponse = ref({
 	status: OperatorStatus.DEFAULT,
 	name: '',
 	value: '',
 	traceback: ''
 });
 
+const executeResponseTraceback = ref('');
+
 const updateLlmQuery = (query: string) => {
 	llmThoughts.value = [];
 	llmQuery.value = query;
@@ -205,7 +209,9 @@ const runCode = () => {
 				executedCode = data.content.code;
 			})
 			.register('stream', (data) => {
-				console.log('stream', data);
+				if (data?.content?.name === 'stderr' || data?.content?.name === 'stdout') {
+					executeResponseTraceback.value = `${executeResponseTraceback.value} ${data.content.text}`;
+				}
 			})
 			.register('model_preview', (data) => {
 				if (!data.content) return;
@@ -219,7 +225,7 @@ const runCode = () => {
 				let status = OperatorStatus.DEFAULT;
 				if (data.msg.content.status === 'ok') status = OperatorStatus.SUCCESS;
 				if (data.msg.content.status === 'error') status = OperatorStatus.ERROR;
-				executeResponse.value = {
+				executeErrorResponse.value = {
 					status,
 					name: data.msg.content.ename ? data.msg.content.ename : '',
 					value: data.msg.content.evalue ? data.msg.content.evalue : '',

From 354058221b85e023a90129ebb7274d273f577a0b Mon Sep 17 00:00:00 2001
From: Adrian <95376249+asylves1@users.noreply.github.com>
Date: Wed, 9 Oct 2024 10:23:24 -0400
Subject: [PATCH 2/6] Add titles to generated compare models graphs (#5100)

---
 .../model-comparison-operation.ts             |  4 ++-
 .../tera-model-comparison.vue                 | 27 ++++++++++++++++++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/model-comparison/model-comparison-operation.ts b/packages/client/hmi-client/src/components/workflow/ops/model-comparison/model-comparison-operation.ts
index 4f6f99d61d..61e99b0844 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/model-comparison/model-comparison-operation.ts
+++ b/packages/client/hmi-client/src/components/workflow/ops/model-comparison/model-comparison-operation.ts
@@ -8,6 +8,7 @@ export interface ModelComparisonOperationState extends BaseState {
 	notebookHistory: NotebookHistory[];
 	hasCodeRun: boolean;
 	comparisonImageIds: string[];
+	comparisonPairs: string[][];
 }
 
 export const ModelComparisonOperation: Operation = {
@@ -26,7 +27,8 @@ export const ModelComparisonOperation: Operation = {
 		const init: ModelComparisonOperationState = {
 			notebookHistory: [],
 			hasCodeRun: false,
-			comparisonImageIds: []
+			comparisonImageIds: [],
+			comparisonPairs: []
 		};
 		return init;
 	}
diff --git a/packages/client/hmi-client/src/components/workflow/ops/model-comparison/tera-model-comparison.vue b/packages/client/hmi-client/src/components/workflow/ops/model-comparison/tera-model-comparison.vue
index 1b5adef563..663e58253f 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/model-comparison/tera-model-comparison.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/model-comparison/tera-model-comparison.vue
@@ -104,7 +104,7 @@
 				/>
 				<ul>
 					<li v-for="(image, index) in structuralComparisons" :key="index">
-						<label>Comparison {{ index + 1 }}</label>
+						<label>Comparison {{ index + 1 }}: {{ getTitle(index) }}</label>
 						<Image id="img" :src="image" :alt="`Structural comparison ${index + 1}`" preview />
 					</li>
 				</ul>
@@ -198,6 +198,7 @@ const code = ref(props.node.state.notebookHistory?.[0]?.code ?? '');
 const llmThoughts = ref<any[]>([]);
 const isKernelReady = ref(false);
 const contextLanguage = ref<string>('python3');
+const comparisonPairs = ref(props.node.state.comparisonPairs);
 
 const initializeAceEditor = (editorInstance: any) => {
 	editor = editorInstance;
@@ -218,6 +219,18 @@ function updateImagesState(operationType: string, newImageId: string | null = nu
 	emit('update-state', state);
 }
 
+function updateComparisonTitlesState(operationType: string, pairs: string[][] | null = null) {
+	const state = cloneDeep(props.node.state);
+	if (operationType === 'add' && pairs !== null) state.comparisonPairs = pairs;
+	else if (operationType === 'clear') state.comparisonPairs = [];
+	emit('update-state', state);
+}
+
+function getTitle(index: number) {
+	if (!comparisonPairs.value[index]) return '';
+	return `${comparisonPairs.value[index][0].replaceAll('_', ' ')} VS ${comparisonPairs.value[index][1].replaceAll('_', ' ')}`;
+}
+
 function updateCodeState() {
 	const state = saveCodeToState(props.node, code.value, true);
 	emit('update-state', state);
@@ -226,6 +239,7 @@ function updateCodeState() {
 function emptyImages() {
 	deleteImages(props.node.state.comparisonImageIds); // Delete images from S3
 	updateImagesState('clear'); // Then their ids can be removed from the state
+	updateComparisonTitlesState('clear');
 	structuralComparisons.value = [];
 }
 
@@ -248,6 +262,17 @@ function runCode() {
 	emptyImages();
 	updateCodeState();
 
+	kernelManager.sendMessage('get_comparison_pairs_request', {}).register('any_get_comparison_pairs_reply', (data) => {
+		const pairs = data.msg.content?.return?.comparison_pairs;
+		const state = cloneDeep(props.node.state);
+		if (pairs.length) {
+			updateComparisonTitlesState('add', pairs);
+			comparisonPairs.value = pairs;
+		} else if (state.comparisonPairs.length) {
+			comparisonPairs.value = state.comparisonPairs;
+		}
+	});
+
 	kernelManager
 		.sendMessage('execute_request', messageContent)
 		.register('display_data', (data) => {

From f1fc1aae1cc4a786ed5d6c50ba550b06b13e92e7 Mon Sep 17 00:00:00 2001
From: Derek Vince <dvince@uncharted.software>
Date: Wed, 9 Oct 2024 11:08:09 -0400
Subject: [PATCH 3/6] [BUG]: Incorrect column info in the Dataset page of an
 uploaded CSV file with `,` in column headers #4994 (#5093)

---
 .../dataset/tera-dataset-datatable.vue        |   2 +-
 packages/server/build.gradle                  |   5 +-
 packages/server/gradle.lockfile               |   6 +-
 .../dataservice/DatasetController.java        | 225 +++++++++---------
 .../models/dataservice/dataset/Dataset.java   |   2 +
 .../service/data/DatasetService.java          | 196 ++++++++-------
 .../src/main/resources/messages.properties    |   3 +
 7 files changed, 244 insertions(+), 195 deletions(-)

diff --git a/packages/client/hmi-client/src/components/dataset/tera-dataset-datatable.vue b/packages/client/hmi-client/src/components/dataset/tera-dataset-datatable.vue
index 5ef60e65bf..931e69ec87 100644
--- a/packages/client/hmi-client/src/components/dataset/tera-dataset-datatable.vue
+++ b/packages/client/hmi-client/src/components/dataset/tera-dataset-datatable.vue
@@ -36,7 +36,7 @@
 		<!-- Datable -->
 		<DataTable
 			:class="previewMode ? 'p-datatable-xsm' : 'p-datatable-sm'"
-			:value="rawContent.csv.slice(1, rawContent.csv.length)"
+			:value="rawContent.csv"
 			:rows="rows"
 			paginator
 			:paginatorPosition="paginatorPosition ? paginatorPosition : `bottom`"
diff --git a/packages/server/build.gradle b/packages/server/build.gradle
index 8c0853979f..9c0de41dbb 100644
--- a/packages/server/build.gradle
+++ b/packages/server/build.gradle
@@ -63,9 +63,10 @@ dependencies {
     implementation 'io.micrometer:micrometer-registry-prometheus'
     implementation 'javax.annotation:javax.annotation-api:1.3.2'
     implementation 'javax.inject:javax.inject:1'
-    implementation 'org.apache.commons:commons-csv:1.10.0'
+    implementation 'org.apache.commons:commons-csv:1.12.0'
     implementation 'org.apache.commons:commons-fileupload2-jakarta:2.0.0-M1'
-    implementation 'org.apache.commons:commons-text:1.10.0'
+    implementation 'org.apache.commons:commons-text:1.12.0'
+    implementation 'commons-io:commons-io:2.17.0'
     implementation 'org.eclipse.microprofile.jwt:microprofile-jwt-auth-api:2.1'
     implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.24'
     implementation 'org.flywaydb:flyway-core'
diff --git a/packages/server/gradle.lockfile b/packages/server/gradle.lockfile
index 8a822f28c1..3abcc4020b 100644
--- a/packages/server/gradle.lockfile
+++ b/packages/server/gradle.lockfile
@@ -67,7 +67,7 @@ com.vaadin.external.google:android-json:0.0.20131108.vaadin1=aotTestCompileClass
 com.zaxxer:HikariCP:5.0.1=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 commons-codec:commons-codec:1.15=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 commons-fileupload:commons-fileupload:1.5=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-commons-io:commons-io:2.13.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+commons-io:commons-io:2.17.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 edu.ucar:cdm-core:5.5.3=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 edu.ucar:httpservices:5.5.3=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 edu.ucar:udunits:5.5.3=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -160,11 +160,11 @@ net.minidev:accessors-smart:2.4.11=aotTestCompileClasspath,nativeImageTestClassp
 net.minidev:json-smart:2.4.11=aotTestCompileClasspath,nativeImageTestClasspath,processTestAotClasspath,testCompileClasspath,testRuntimeClasspath
 net.sf.jopt-simple:jopt-simple:5.0.2=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.antlr:antlr4-runtime:4.10.1=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.commons:commons-csv:1.10.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.commons:commons-csv:1.12.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-fileupload2-core:2.0.0-M1=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-fileupload2-jakarta:2.0.0-M1=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-lang3:3.12.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.commons:commons-text:1.10.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.commons:commons-text:1.12.0=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.httpcomponents:httpasyncclient:4.1.5=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.httpcomponents:httpclient:4.5.14=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.httpcomponents:httpcore-nio:4.4.16=aotCompileClasspath,aotTestCompileClasspath,compileClasspath,nativeImageClasspath,nativeImageTestClasspath,processAotClasspath,processTestAotClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java
index aaf0a9904c..001e82d817 100644
--- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java
+++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java
@@ -1,16 +1,16 @@
 package software.uncharted.terarium.hmiserver.controller.dataservice;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.math.Quantiles;
-import com.google.common.math.Stats;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -18,6 +18,9 @@
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
@@ -105,9 +108,11 @@ public ResponseEntity<Dataset> createDataset(
 		try {
 			return ResponseEntity.status(HttpStatus.CREATED).body(datasetService.createAsset(dataset, projectId, permission));
 		} catch (final IOException e) {
-			final String error = "Unable to create dataset";
-			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			log.error("Unable to create dataset", e);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 	}
 
@@ -156,9 +161,11 @@ public ResponseEntity<Dataset> getDataset(
 
 			return dataset.map(ResponseEntity::ok).orElseGet(() -> ResponseEntity.notFound().build());
 		} catch (final Exception e) {
-			final String error = "Unable to get dataset";
-			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			log.error("Unable to get dataset", e);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 	}
 
@@ -193,9 +200,11 @@ public ResponseEntity<ResponseDeleted> deleteDataset(
 			datasetService.deleteAsset(id, projectId, permission);
 			return ResponseEntity.ok(new ResponseDeleted("Dataset", id));
 		} catch (final IOException e) {
-			final String error = "Unable to delete dataset";
-			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			log.error("Unable to delete dataset", e);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("dataset.unable-to-delete")
+			);
 		}
 	}
 
@@ -231,9 +240,11 @@ ResponseEntity<Dataset> updateDataset(
 			final Optional<Dataset> updated = datasetService.updateAsset(dataset, projectId, permission);
 			return updated.map(ResponseEntity::ok).orElseGet(() -> ResponseEntity.notFound().build());
 		} catch (final IOException e) {
-			final String error = "Unable to update a dataset";
-			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			log.error("Unable to update a dataset", e);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("dataset.unable-to-update")
+			);
 		}
 	}
 
@@ -262,33 +273,41 @@ public ResponseEntity<CsvAsset> getCsv(
 		@RequestParam("filename") final String filename,
 		@RequestParam(name = "limit", defaultValue = "" + DEFAULT_CSV_LIMIT, required = false) final Integer limit
 	) {
-		final List<List<String>> csv;
+		final CSVParser csvParser;
 		try {
-			csv = datasetService.getCSVFile(filename, datasetId, limit);
-			if (csv == null) {
-				final String error = "Unable to get CSV";
-				log.error(error);
-				throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			csvParser = datasetService.getCSVFileParser(filename, datasetId);
+			if (csvParser == null) {
+				log.error("Unable to get CSV");
+				throw new ResponseStatusException(
+					org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+					messages.get("dataset.not-found")
+				);
 			}
 		} catch (final IOException e) {
-			final String error = "Unable to parse CSV";
-			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			log.error("Unable to parse CSV", e);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 
-		final List<String> headers = csv.get(0);
-		final List<CsvColumnStats> csvColumnStats = new ArrayList<>();
-		for (int i = 0; i < csv.get(0).size(); i++) {
-			final List<String> column = getColumn(csv, i);
-			csvColumnStats.add(getStats(column.subList(1, column.size()))); // remove first as it is header:
-		}
+		// We have a parser over our CSV file. Now for the front end we need to create a matrix of strings
+		// to represent the CSV file up to our limit. Then we need to calculate the column statistics.
 
-		final int linesToRead = limit != null ? (limit == -1 ? csv.size() : limit) : DEFAULT_CSV_LIMIT;
+		final List<List<String>> csv = csvParser
+			.getRecords()
+			.stream()
+			.limit(limit >= 0 ? limit : Long.MAX_VALUE)
+			.map(record -> new ArrayList<>(record.toList()))
+			.collect(Collectors.toList());
+
+		// TODO - this should be done on csv post/push, and in task to handle large files.
+		final List<CsvColumnStats> csvColumnStats = DatasetService.calculateColumnStatistics(csv);
 
 		final CsvAsset csvAsset = new CsvAsset(
-			csv.subList(0, Math.min(linesToRead + 1, csv.size())),
+			csv,
 			csvColumnStats,
-			headers,
+			new ArrayList<>(csvParser.getHeaderMap().keySet()),
 			csv.size()
 		);
 
@@ -365,12 +384,18 @@ public ResponseEntity<PresignedURL> getDownloadURL(
 		try {
 			dataset = datasetService.getAsset(id, permission);
 			if (dataset.isEmpty()) {
-				throw new ResponseStatusException(org.springframework.http.HttpStatus.NOT_FOUND, "Dataset not found");
+				throw new ResponseStatusException(
+					org.springframework.http.HttpStatus.NOT_FOUND,
+					messages.get("dataset.not-found")
+				);
 			}
 		} catch (final Exception e) {
 			final String error = "Unable to get dataset";
 			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 
 		if (
@@ -390,7 +415,10 @@ public ResponseEntity<PresignedURL> getDownloadURL(
 			if (url == null) {
 				final String error = "The file " + filename + " was not found in the dataset";
 				log.error(error);
-				throw new ResponseStatusException(org.springframework.http.HttpStatus.NOT_FOUND, error);
+				throw new ResponseStatusException(
+					org.springframework.http.HttpStatus.NOT_FOUND,
+					messages.get("dataset.file-not-found")
+				);
 			}
 
 			final PresignedURL presigned = new PresignedURL().setUrl(url).setMethod("GET");
@@ -402,13 +430,16 @@ public ResponseEntity<PresignedURL> getDownloadURL(
 			} catch (final Exception e) {
 				final String error = "Unable to get download url";
 				log.error(error, e);
-				throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+				throw new ResponseStatusException(
+					org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+					messages.get("postgres.service-unavailable")
+				);
 			}
 		}
 	}
 
 	/**
-	 * Downloads a CSV file from github given the path and owner name, then uploads
+	 * Uploads a CSV file from github given the path and owner name, then uploads
 	 * it to the dataset.
 	 */
 	@PutMapping("/{id}/upload-csv-from-github")
@@ -447,12 +478,28 @@ public ResponseEntity<ResponseStatus> uploadCsvFromGithub(
 		if (csvString == null) {
 			final String error = "Unable to download csv from github";
 			log.error(error);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("dataset.download-error")
+			);
+		}
+
+		CSVParser csvParser = null;
+		try {
+			csvParser = new CSVParser(
+				new StringReader(csvString),
+				CSVFormat.Builder.create(CSVFormat.DEFAULT).setHeader().setSkipHeaderRecord(false).build()
+			);
+		} catch (IOException e) {
+			log.error("Unable to parse csv from github", e);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("dataset.parse-error")
+			);
 		}
 
+		List<String> headers = new ArrayList<>(csvParser.getHeaderMap().keySet());
 		final HttpEntity csvEntity = new StringEntity(csvString, ContentType.APPLICATION_OCTET_STREAM);
-		final String[] csvRows = csvString.split("\\R");
-		final String[] headers = csvRows[0].split(",");
 		return uploadCSVAndUpdateColumns(datasetId, projectId, filename, csvEntity, headers, permission);
 	}
 
@@ -497,19 +544,22 @@ public ResponseEntity<ResponseStatus> uploadCsv(
 			final byte[] csvBytes = input.getBytes();
 
 			final HttpEntity csvEntity = new ByteArrayEntity(csvBytes, ContentType.APPLICATION_OCTET_STREAM);
-			final String csvString = new String(csvBytes);
-			final String[] csvRows = csvString.split("\\R");
-			final String[] headers = csvRows[0].split(",");
-			for (int i = 0; i < headers.length; i++) {
-				// this is very ugly but we're removing opening and closing "'s around these
-				// strings.
-				headers[i] = headers[i].replaceAll("^\"|\"$", "");
-			}
+
+			Reader targetReader = new InputStreamReader(new ByteArrayInputStream(csvBytes));
+			CSVParser csvParser = new CSVParser(
+				targetReader,
+				CSVFormat.Builder.create(CSVFormat.DEFAULT).setHeader().setSkipHeaderRecord(false).build()
+			);
+			List<String> headers = new ArrayList<>(csvParser.getHeaderMap().keySet());
+
 			return uploadCSVAndUpdateColumns(datasetId, projectId, filename, csvEntity, headers, permission);
 		} catch (final IOException e) {
 			final String error = "Unable to upload csv dataset";
 			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 	}
 
@@ -550,7 +600,10 @@ public ResponseEntity<Void> uploadData(
 				if (updatedDataset.isEmpty()) {
 					final String error = "Failed to get dataset after upload";
 					log.error(error);
-					throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+					throw new ResponseStatusException(
+						org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+						messages.get("dataset.not-found")
+					);
 				}
 
 				if (!updatedDataset.get().getFileNames().contains(filename)) {
@@ -573,7 +626,10 @@ public ResponseEntity<Void> uploadData(
 		} catch (final IOException e) {
 			final String error = "Unable to upload file to dataset";
 			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 	}
 
@@ -606,7 +662,10 @@ public ResponseEntity<PresignedURL> getUploadURL(
 		} catch (final Exception e) {
 			final String error = "Unable to get upload url";
 			log.error(error, e);
-			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
+			throw new ResponseStatusException(
+				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
+				messages.get("postgres.service-unavailable")
+			);
 		}
 	}
 
@@ -631,7 +690,7 @@ private ResponseEntity<ResponseStatus> uploadCSVAndUpdateColumns(
 		final UUID projectId,
 		final String filename,
 		final HttpEntity csvEntity,
-		final String[] headers,
+		final List<String> headers,
 		final Schema.Permission hasWritePermission
 	) {
 		try {
@@ -640,7 +699,7 @@ private ResponseEntity<ResponseStatus> uploadCSVAndUpdateColumns(
 
 			// update dataset with headers if the previous upload was successful
 			if (status == HttpStatus.OK.value()) {
-				log.debug("Successfully uploaded CSV file to dataset {}. Now updating TDS with headers", datasetId);
+				log.debug("Successfully uploaded CSV file to dataset {}. Now updating with headers", datasetId);
 
 				final Optional<Dataset> updatedDataset = datasetService.getAsset(datasetId, hasWritePermission);
 				if (updatedDataset.isEmpty()) {
@@ -648,7 +707,7 @@ private ResponseEntity<ResponseStatus> uploadCSVAndUpdateColumns(
 					return ResponseEntity.internalServerError().build();
 				}
 
-				DatasetService.addDatasetColumns(updatedDataset.get(), filename, Arrays.asList(headers));
+				DatasetService.addDatasetColumns(updatedDataset.get(), filename, headers);
 
 				// add the filename to existing file names
 				if (!updatedDataset.get().getFileNames().contains(filename)) {
@@ -663,61 +722,11 @@ private ResponseEntity<ResponseStatus> uploadCSVAndUpdateColumns(
 			log.error("Unable to upload csv data", e);
 			throw new ResponseStatusException(
 				org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR,
-				"Unable to PUT csv data"
+				messages.get("postgres.service-unavailable")
 			);
 		}
 	}
 
-	private static List<String> getColumn(final List<List<String>> matrix, final int columnNumber) {
-		final List<String> column = new ArrayList<>();
-		for (final List<String> strings : matrix) {
-			if (strings.size() > columnNumber) {
-				column.add(strings.get(columnNumber));
-			}
-		}
-		return column;
-	}
-
-	/**
-	 * Given a column and an amount of bins creates a CsvColumnStats object.
-	 *
-	 * @param aCol column to get stats for
-	 * @return CsvColumnStats object
-	 */
-	private static CsvColumnStats getStats(final List<String> aCol) {
-		final List<Integer> bins = new ArrayList<>();
-		try {
-			// set up row as numbers. may fail here.
-			// List<Integer> numberList = aCol.stream().map(String s ->
-			// Integer.parseInt(s.trim()));
-			final List<Double> numberList = aCol.stream().map(Double::valueOf).collect(Collectors.toList());
-			Collections.sort(numberList);
-			final double minValue = numberList.get(0);
-			final double maxValue = numberList.get(numberList.size() - 1);
-			final double meanValue = Stats.meanOf(numberList);
-			final double medianValue = Quantiles.median().compute(numberList);
-			final double sdValue = Stats.of(numberList).populationStandardDeviation();
-			final int binCount = 10;
-			// Set up bins
-			for (int i = 0; i < binCount; i++) {
-				bins.add(0);
-			}
-			final double stepSize = (numberList.get(numberList.size() - 1) - numberList.get(0)) / (binCount - 1);
-
-			// Fill bins:
-			for (final Double aDouble : numberList) {
-				final int index = (int) Math.abs(Math.floor((aDouble - numberList.get(0)) / stepSize));
-				final Integer value = bins.get(index);
-				bins.set(index, value + 1);
-			}
-
-			return new CsvColumnStats(bins, minValue, maxValue, meanValue, medianValue, sdValue);
-		} catch (final Exception e) {
-			// Cannot convert column to double, just return empty list.
-			return new CsvColumnStats(bins, 0, 0, 0, 0, 0);
-		}
-	}
-
 	@GetMapping("/{id}/preview")
 	@Secured(Roles.USER)
 	@Operation(summary = "Gets a preview of the data asset")
diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java
index 0b96dd0c63..13696e6390 100644
--- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java
+++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java
@@ -32,6 +32,8 @@
 @Entity
 public class Dataset extends TerariumAsset {
 
+	public static final String NC_EXTENSION = ".nc";
+
 	@Serial
 	private static final long serialVersionUID = 6927286281160755696L;
 
diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java
index 19d9ed14b8..b6f5fae5dc 100644
--- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java
+++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java
@@ -1,23 +1,26 @@
 package software.uncharted.terarium.hmiserver.service.data;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.math.Quantiles;
+import com.google.common.math.Stats;
 import io.micrometer.observation.annotation.Observed;
-import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.StringReader;
+import java.io.Reader;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
 import org.springframework.stereotype.Service;
 import software.uncharted.terarium.hmiserver.configuration.Config;
 import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration;
+import software.uncharted.terarium.hmiserver.models.dataservice.CsvColumnStats;
 import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset;
 import software.uncharted.terarium.hmiserver.models.dataservice.dataset.DatasetColumn;
 import software.uncharted.terarium.hmiserver.repository.data.DatasetRepository;
@@ -73,13 +76,8 @@ public String getAssetAlias() {
 	@Observed(name = "function_profile")
 	public Dataset createAsset(final Dataset asset, final UUID projectId, final Schema.Permission hasWritePermission)
 		throws IOException {
-		extractColumnsForFreshCreate(asset);
-
-		if (asset.getColumns() != null) {
-			for (final DatasetColumn column : asset.getColumns()) {
-				column.setDataset(asset);
-			}
-		}
+		extractColumns(asset);
+		verifyColumnRelationship(asset);
 		return super.createAsset(asset, projectId, hasWritePermission);
 	}
 
@@ -91,13 +89,8 @@ public List<Dataset> createAssets(
 		final Schema.Permission hasWritePermission
 	) throws IOException {
 		for (final Dataset asset : assets) {
-			if (asset.getColumns() != null) {
-				for (final DatasetColumn column : asset.getColumns()) {
-					column.setDataset(asset);
-				}
-			}
+			verifyColumnRelationship(asset);
 		}
-
 		return super.createAssets(assets, projectId, hasWritePermission);
 	}
 
@@ -108,11 +101,7 @@ public Optional<Dataset> updateAsset(
 		final UUID projectId,
 		final Schema.Permission hasWritePermission
 	) throws IOException, IllegalArgumentException {
-		if (asset.getColumns() != null) {
-			for (final DatasetColumn column : asset.getColumns()) {
-				column.setDataset(asset);
-			}
-		}
+		verifyColumnRelationship(asset);
 		return super.updateAsset(asset, projectId, hasWritePermission);
 	}
 
@@ -120,52 +109,41 @@ public Optional<Dataset> updateAsset(
 	@Observed(name = "function_profile")
 	public Dataset extractColumnsFromFiles(final Dataset dataset) throws IOException {
 		if (dataset.getColumns() != null && !dataset.getColumns().isEmpty()) {
-			// columns are set. No need to extract
 			return dataset;
 		}
-		if (dataset.getFileNames() != null || dataset.getFileNames().isEmpty()) {
-			// no file names to extract columns from
-			return dataset;
-		}
-
-		for (final String filename : dataset.getFileNames()) {
-			if (!filename.endsWith(".nc")) {
-				final List<List<String>> csv = getCSVFile(filename, dataset.getId(), 1);
-				if (csv == null || csv.isEmpty()) {
-					continue;
-				}
-				addDatasetColumns(dataset, filename, csv.get(0));
-			}
-		}
-
-		return dataset;
+		return extractColumns(dataset);
 	}
 
+	/**
+	 * Extract the columns from the dataset files.
+	 *
+	 * @param dataset the dataset to extract the columns from
+	 * @return the dataset with the columns extracted
+	 * @throws IOException if the files cannot be read
+	 */
 	@Observed(name = "function_profile")
-	private Dataset extractColumnsForFreshCreate(final Dataset dataset) throws IOException {
-		if (dataset.getFileNames() != null || dataset.getFileNames().isEmpty()) {
-			// no file names to extract columns from
-			return dataset;
-		}
-
-		for (final String filename : dataset.getFileNames()) {
-			if (!filename.endsWith(".nc")) {
-				try {
-					final List<List<String>> csv = getCSVFile(filename, dataset.getId(), 1);
-					if (csv == null || csv.isEmpty()) {
-						continue;
-					}
-					addDatasetColumns(dataset, filename, csv.get(0));
-				} catch (final IOException e) {
-					// if file is not available yet, don't continue gracefully
+	private Dataset extractColumns(final Dataset dataset) throws IOException {
+		if (dataset.getFileNames() != null) {
+			for (String filename : dataset.getFileNames()) {
+				if (filename.endsWith(Dataset.NC_EXTENSION)) {
 					continue;
 				}
+				CSVParser csvParser = getCSVFileParser(filename, dataset.getId());
+				if (csvParser == null) continue;
+				List<String> headers = new ArrayList<>(csvParser.getHeaderMap().keySet());
+				addDatasetColumns(dataset, filename, headers);
 			}
 		}
-
 		return dataset;
 	}
 
+	/**
+	 * Add the columns to the dataset.
+	 *
+	 * @param dataset the dataset to add the columns to
+	 * @param fileName the name of the file
+	 * @param headers the headers to add
+	 */
 	@Observed(name = "function_profile")
 	public static void addDatasetColumns(final Dataset dataset, final String fileName, final List<String> headers) {
 		if (dataset.getColumns() == null) {
@@ -181,43 +159,99 @@ public static void addDatasetColumns(final Dataset dataset, final String fileNam
 		}
 	}
 
+	/**
+	 * Get the CSV file parser for the given file name and dataset id.
+	 *
+	 * @param filename  the name of the file to parse
+	 * @param datasetId the id of the dataset
+	 * @return the CSV file parser
+	 * @throws IOException if the file cannot be read
+	 */
 	@Observed(name = "function_profile")
-	public List<List<String>> getCSVFile(final String filename, final UUID datasetId, final Integer limit)
-		throws IOException {
-		String rawCSV = "";
-
+	public CSVParser getCSVFileParser(final String filename, final UUID datasetId) throws IOException {
 		final Optional<byte[]> bytes = fetchFileAsBytes(datasetId, filename);
 		if (bytes.isEmpty()) {
 			return null;
 		}
 
-		final BufferedReader reader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(bytes.get())));
+		final Reader reader = new InputStreamReader(new ByteArrayInputStream(bytes.get()));
+		return new CSVParser(
+			reader,
+			CSVFormat.Builder.create(CSVFormat.DEFAULT).setHeader().setSkipHeaderRecord(false).build()
+		);
+	}
 
-		String line;
-		Integer count = 0;
-		while ((line = reader.readLine()) != null) {
-			if (limit > 0 && count > limit) {
-				break;
-			}
-			rawCSV += line + '\n';
-			count++;
-		}
+	/**
+	 * Verify that the columns have the correct relationship to the dataset. This is needed
+	 * because the columns are not directly related to the dataset in the database by default.
+	 *
+	 * @param asset the dataset to verify
+	 */
+	private static void verifyColumnRelationship(Dataset asset) {
+		Optional.ofNullable(asset.getColumns()).ifPresent(columns -> columns.forEach(column -> column.setDataset(asset)));
+	}
 
-		final List<List<String>> csv;
-		csv = csvToRecords(rawCSV);
-		return csv;
+	/**
+	 * Calculate the statistics for the columns in a CSV.
+	 *
+	 * @param csv the CSV to calculate the statistics for
+	 * @return the statistics for the columns
+	 */
+	public static List<CsvColumnStats> calculateColumnStatistics(List<List<String>> csv) {
+		List<CsvColumnStats> stats = new ArrayList<>();
+		csv.get(0).forEach(column -> stats.add(getStats(getColumn(csv, csv.get(0).indexOf(column)))));
+		return stats;
 	}
 
-	@Observed(name = "function_profile")
-	private static List<List<String>> csvToRecords(final String rawCsvString) throws IOException {
-		final List<List<String>> records = new ArrayList<>();
-		try (final CSVParser parser = new CSVParser(new StringReader(rawCsvString), CSVFormat.DEFAULT)) {
-			for (final CSVRecord csvRecord : parser) {
-				final List<String> values = new ArrayList<>();
-				csvRecord.forEach(values::add);
-				records.add(values);
+	/**
+	 * Get a column from a CSV matrix.
+	 *
+	 * @param matrix       the matrix to get the column from
+	 * @param columnNumber the number of the column to get
+	 * @return the column
+	 */
+	private static List<String> getColumn(final List<List<String>> matrix, final int columnNumber) {
+		return matrix
+			.stream()
+			.filter(strings -> strings.size() > columnNumber)
+			.map(strings -> strings.get(columnNumber))
+			.collect(Collectors.toList());
+	}
+
+	/**
+	 * Given a column and an amount of bins creates a CsvColumnStats object.
+	 *
+	 * @param aCol column to get stats for
+	 * @return CsvColumnStats object
+	 */
+	private static CsvColumnStats getStats(final List<String> aCol) {
+		final List<Integer> bins = new ArrayList<>();
+		try {
+			final List<Double> numberList = aCol.stream().map(Double::valueOf).collect(Collectors.toList());
+			Collections.sort(numberList);
+			final double minValue = numberList.get(0);
+			final double maxValue = numberList.get(numberList.size() - 1);
+			final double meanValue = Stats.meanOf(numberList);
+			final double medianValue = Quantiles.median().compute(numberList);
+			final double sdValue = Stats.of(numberList).populationStandardDeviation();
+			final int binCount = 10;
+			// Set up bins
+			for (int i = 0; i < binCount; i++) {
+				bins.add(0);
+			}
+			final double stepSize = (numberList.get(numberList.size() - 1) - numberList.get(0)) / (binCount - 1);
+
+			// Fill bins:
+			for (final Double aDouble : numberList) {
+				final int index = (int) Math.abs(Math.floor((aDouble - numberList.get(0)) / stepSize));
+				final Integer value = bins.get(index);
+				bins.set(index, value + 1);
 			}
+
+			return new CsvColumnStats(bins, minValue, maxValue, meanValue, medianValue, sdValue);
+		} catch (final Exception e) {
+			// Cannot convert column to double, just return empty list.
+			return new CsvColumnStats(bins, 0, 0, 0, 0, 0);
 		}
-		return records;
 	}
 }
diff --git a/packages/server/src/main/resources/messages.properties b/packages/server/src/main/resources/messages.properties
index 6c8df63f34..43fef9ea86 100644
--- a/packages/server/src/main/resources/messages.properties
+++ b/packages/server/src/main/resources/messages.properties
@@ -23,6 +23,9 @@ dataset.not-found = We couldn't find the requested dataset. Please verify that t
 dataset.unable-to-delete = We couldn't delete the dataset. Please try again later.
 dataset.unable-to-update = We couldn't update the dataset. Please try again later.
 dataset.files.not-found = We couldn't find any files associated with this dataset. The service may be unavailable, or the dataset may be corrupted. Please upload the dataset again or contact support.
+dataset.parse-error = We couldn't import the dataset. Please verify that the dataset is correctly formatted and try again.
+dataset.file-not-found = We couldn't find the requested file. Please verify that the file you selected is correct or contact support.
+dataset.download-error = We couldn't download the dataset. Please try again later.
 
 document.extraction.not-done = We can't complete this task because the document extraction is still in progress. Please try again when the extraction has finished.
 document.not-found = We couldn't find the requested document. Please verify that the dataset you selected is correct or contact support.

From 794e55871dc56ae2d452cc8d091acf152f30f247 Mon Sep 17 00:00:00 2001
From: Kevin Birk <birk.kevin@gmail.com>
Date: Wed, 9 Oct 2024 11:52:35 -0400
Subject: [PATCH 4/6] Add wait flag for extractions. (#5101)

---
 .../knowledge/KnowledgeController.java           | 16 +++++++++++++---
 .../src/main/resources/application.properties    |  1 +
 .../src/main/resources/messages.properties       |  1 +
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java
index a1c85f941d..1cbb88e7c4 100644
--- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java
+++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java
@@ -21,6 +21,7 @@
 import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeoutException;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -797,14 +798,23 @@ public ResponseEntity<Void> variableExtractions(
 	)
 	public ResponseEntity<Void> pdfExtractions(
 		@RequestParam("document-id") final UUID documentId,
-		@RequestParam(name = "domain", defaultValue = "epi") final String domain,
-		@RequestParam(name = "project-id", required = false) final UUID projectId
+		@RequestParam(name = "project-id", required = false) final UUID projectId,
+		@RequestParam(name = "wait", required = false, defaultValue = "false") final Boolean wait
 	) {
 		final Schema.Permission permission = projectService.checkPermissionCanWrite(
 			currentUserService.get().getId(),
 			projectId
 		);
-		extractionService.extractPDFAndApplyToDocument(documentId, projectId, permission);
+
+		final Future<DocumentAsset> f = extractionService.extractPDFAndApplyToDocument(documentId, projectId, permission);
+		if (wait) {
+			try {
+				f.get();
+			} catch (InterruptedException | ExecutionException e) {
+				log.error("Error extracting PDF", e);
+				throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("document.extracton.failed"));
+			}
+		}
 		return ResponseEntity.accepted().build();
 	}
 
diff --git a/packages/server/src/main/resources/application.properties b/packages/server/src/main/resources/application.properties
index 6696ce873c..98c38bbb01 100644
--- a/packages/server/src/main/resources/application.properties
+++ b/packages/server/src/main/resources/application.properties
@@ -87,6 +87,7 @@ terarium.serviceRequestPatterns[11]=/simulations/**
 terarium.serviceRequestPatterns[12]=/code-asset/**
 terarium.serviceRequestPatterns[13]=/workflows/**
 terarium.serviceRequestPatterns[14]=/interventions/**
+terarium.serviceRequestPatterns[15]=/knowledge/**
 terarium.extractionService.poolSize=10
 terarium.cacheHeadersMaxAge=86400
 management.endpoints.web.exposure.include=health,prometheus
diff --git a/packages/server/src/main/resources/messages.properties b/packages/server/src/main/resources/messages.properties
index 43fef9ea86..c6a9fb48ca 100644
--- a/packages/server/src/main/resources/messages.properties
+++ b/packages/server/src/main/resources/messages.properties
@@ -32,6 +32,7 @@ document.not-found = We couldn't find the requested document. Please verify that
 document.unable-to-delete = We couldn't delete the document. Please try again later.
 document.unable-to-update = We couldn't update the document. Please try again later.
 document.text-length-exceeded = This document exceeds our 600,000-word limit. If you still need to analyze this document, contact support.
+document.extracton.failed = Document extraction failed. Please try again later.
 
 mira.concept.bad-curies = Our domain knowledge service (MIRA) couldn't find any valid concepts for the selected IDs. Please review the IDs that you selected and try again.
 mira.concept.bad-query = Our domain knowledge service (MIRA) couldn't find any valid concepts for the entered query. Please review your search criteria and try again.

From 31145699fb92653121b9e7409f5ad2303347e146 Mon Sep 17 00:00:00 2001
From: Yohann Paris <github@yohannparis.com>
Date: Wed, 9 Oct 2024 12:00:32 -0400
Subject: [PATCH 5/6] 5016 bug when extracting a configuration we need to
 distinguish missing values from 0 (#5066)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 .../tera-initial-other-value-modal.vue        | 107 +++++++-----------
 .../model/petrinet/tera-initial-table.vue     |  50 ++++----
 .../tera-model-config-drilldown.vue           |  66 +++++++----
 .../tera-model-configuration-item.vue         |  56 +++++----
 .../prompts/config_from_document.py           |   4 +-
 5 files changed, 143 insertions(+), 140 deletions(-)

diff --git a/packages/client/hmi-client/src/components/model/petrinet/tera-initial-other-value-modal.vue b/packages/client/hmi-client/src/components/model/petrinet/tera-initial-other-value-modal.vue
index ce54438630..de61c01a92 100644
--- a/packages/client/hmi-client/src/components/model/petrinet/tera-initial-other-value-modal.vue
+++ b/packages/client/hmi-client/src/components/model/petrinet/tera-initial-other-value-modal.vue
@@ -4,45 +4,37 @@
 			<h4>Other configuration values for '{{ otherValueList[0].target }}'</h4>
 		</template>
 		<DataTable
-			:value="otherValueList"
-			@update:selection="onCustomSelectionChange"
 			dataKey="id"
-			:rowsPerPageOptions="[10, 20, 50]"
 			tableStyle="min-width: 55rem"
+			:rowsPerPageOptions="[10, 20, 50]"
+			:value="otherValueList"
+			@update:selection="onCustomSelectionChange"
 		>
 			<template #header> </template>
-			<Column headerStyle="width: 2rem">
+			<Column>
 				<template #body="{ data }">
 					<RadioButton
 						v-model="customSelection"
+						variant="filled"
 						:inputId="data.id"
 						:value="data"
-						variant="filled"
 						@change="onCustomSelectionChange(data)"
 					/>
 				</template>
 			</Column>
-			<Column
-				v-for="(col, index) in selectedColumns"
-				:field="col.field"
-				:header="col.header"
-				:sortable="col.field !== 'stats'"
-				:key="index"
-				:style="`width: ${getColumnWidth(col.field)}%`"
-			>
+			<Column sortable>
+				<template #header>Configuration name</template>
+				<template #body="{ data }">{{ data.name }}</template>
+			</Column>
+			<Column sortable>
+				<template #header>Source</template>
+				<template #body="{ data }">{{ data.source }}</template>
+			</Column>
+			<Column sortable>
+				<template #header>Value</template>
 				<template #body="{ data }">
-					<template v-if="col.field === 'name'">
-						{{ data.name }}
-					</template>
-					<template v-if="col.field === 'source'">
-						{{ data.source }}
-					</template>
-					<template v-if="col.field === 'expression'">
-						<section class="inline-flex gap-1">
-							<span class="value-label">Constants</span>
-							<span class="value">{{ numberToNist(data.expression) || data.expression }}</span>
-						</section>
-					</template>
+					<span class="value-label">Constants</span>
+					{{ numberToNist(data.expression) || data.expression }}
 				</template>
 			</Column>
 			<ColumnGroup type="footer">
@@ -85,25 +77,25 @@
 			</ColumnGroup>
 		</DataTable>
 		<template #footer>
-			<Button label="Apply selected value" @click="applySelectedValue" :disabled="!selection" />
+			<Button label="Apply selected value" :disabled="!selection" @click="applySelectedValue" />
 			<Button label="Cancel" severity="secondary" outlined @click="emit('close-modal')" />
 		</template>
 	</tera-modal>
 </template>
 
 <script setup lang="ts">
-import { numberToNist } from '@/utils/number';
-import TeraInputText from '@/components/widgets/tera-input-text.vue';
-import TeraInputNumber from '@/components/widgets/tera-input-number.vue';
-import { DistributionType } from '@/services/distribution';
 import { ref } from 'vue';
 import Button from 'primevue/button';
-import TeraModal from '@/components/widgets/tera-modal.vue';
-import DataTable from 'primevue/datatable';
 import Column from 'primevue/column';
-import Row from 'primevue/row';
+import DataTable from 'primevue/datatable';
 import ColumnGroup from 'primevue/columngroup';
 import RadioButton from 'primevue/radiobutton';
+import Row from 'primevue/row';
+import TeraInputNumber from '@/components/widgets/tera-input-number.vue';
+import TeraInputText from '@/components/widgets/tera-input-text.vue';
+import TeraModal from '@/components/widgets/tera-modal.vue';
+import { numberToNist } from '@/utils/number';
+import { DistributionType } from '@/services/distribution';
 
 const props = defineProps<{
 	id: string;
@@ -112,11 +104,6 @@ const props = defineProps<{
 }>();
 
 const otherValueList = ref(props.otherValueList);
-const columns = ref([
-	{ field: 'name', header: 'Configuration name' },
-	{ field: 'source', header: 'Source' },
-	{ field: 'expression', header: 'Value' }
-]);
 
 const emit = defineEmits(['update-expression', 'update-source', 'close-modal']);
 
@@ -126,7 +113,6 @@ const customConstant = ref(0);
 
 const numberOptions = [DistributionType.Constant, DistributionType.Uniform];
 
-const selectedColumns = ref(columns.value);
 const customSelection = ref(false);
 const selection = ref<null | { id?: string; source?: string; constant?: number }>(null);
 
@@ -141,19 +127,6 @@ const onCustomSelectionChange = (val) => {
 	}
 };
 
-function getColumnWidth(columnField: string) {
-	switch (columnField) {
-		case 'name':
-			return 40;
-		case 'source':
-			return 20;
-		case 'expression':
-			return 100;
-		default:
-			return 100;
-	}
-}
-
 function applySelectedValue() {
 	emit('update-expression', { id: props.id, value: selection.value?.constant });
 	emit('update-source', { id: props.id, value: selection.value?.source });
@@ -162,26 +135,34 @@ function applySelectedValue() {
 </script>
 
 <style scoped>
-.value-label {
-	color: var(--surface-600);
-	padding-right: 1rem;
+/* Override PrimeVue styles */
+:deep(.p-datatable-table) th {
+	padding: var(--gap-4);
+	width: 100%;
 }
-.value {
-	color: var(--surface-900);
+:deep(.p-datatable-table) th:nth-of-type(1) {
+	width: 2rem;
+}
+:deep(.p-datatable-table) th:nth-of-type(2) {
+	width: 40%;
+}
+:deep(.p-datatable-table) th:nth-of-type(3) {
+	width: 20%;
+}
+
+.value-label {
+	color: var(--text-color-subdued);
+	padding-right: var(--gap-4);
 }
 
 .custom-input-label {
+	align-items: center;
 	display: flex;
 	justify-content: left;
-	align-items: center;
-	padding-right: 1rem;
+	padding-right: var(--gap-4);
 }
 
 .custom-input {
 	height: 100%;
 }
-
-:deep(input) {
-	margin-top: 1rem;
-}
 </style>
diff --git a/packages/client/hmi-client/src/components/model/petrinet/tera-initial-table.vue b/packages/client/hmi-client/src/components/model/petrinet/tera-initial-table.vue
index 250417a441..9d99911fc1 100644
--- a/packages/client/hmi-client/src/components/model/petrinet/tera-initial-table.vue
+++ b/packages/client/hmi-client/src/components/model/petrinet/tera-initial-table.vue
@@ -6,7 +6,7 @@
 				<tera-input-text v-model="filterText" placeholder="Filter" class="w-2 p-1" />
 			</template>
 
-			<ul class="pl-1">
+			<ul>
 				<li v-for="{ baseInitial, childInitials, isVirtual } in initialList" :key="baseInitial">
 					<!-- Stratified -->
 					<section v-if="isVirtual" class="initial-entry-stratified">
@@ -16,22 +16,19 @@
 									<span>{{ baseInitial }}</span>
 									<Button label="Open Matrix" text size="small" @click.stop="matrixModalId = baseInitial" />
 								</template>
-								<div class="flex">
-									<ul class="ml-1">
-										<li v-for="{ target } in childInitials" :key="target">
-											<tera-initial-entry
-												:model="model"
-												:model-configuration="modelConfiguration"
-												:modelConfigurations="modelConfigurations"
-												:initial-id="target"
-												:feature-config="featureConfig"
-												@update-expression="emit('update-expression', $event)"
-												@update-source="emit('update-source', $event)"
-											/>
-											<Divider type="solid" />
-										</li>
-									</ul>
-								</div>
+								<ul>
+									<li v-for="{ target } in childInitials" :key="target">
+										<tera-initial-entry
+											:model="model"
+											:model-configuration="modelConfiguration"
+											:modelConfigurations="modelConfigurations"
+											:feature-config="featureConfig"
+											:initial-id="target"
+											@update-expression="emit('update-expression', $event)"
+											@update-source="emit('update-source', $event)"
+										/>
+									</li>
+								</ul>
 							</AccordionTab>
 						</Accordion>
 					</section>
@@ -47,7 +44,6 @@
 						@update-expression="emit('update-expression', $event)"
 						@update-source="emit('update-source', $event)"
 					/>
-					<Divider type="solid" />
 				</li>
 			</ul>
 		</AccordionTab>
@@ -74,7 +70,6 @@ import { MiraModel, MiraTemplateParams } from '@/model-representation/mira/mira-
 import Accordion from 'primevue/accordion';
 import AccordionTab from 'primevue/accordiontab';
 import Button from 'primevue/button';
-import Divider from 'primevue/divider';
 import TeraInputText from '@/components/widgets/tera-input-text.vue';
 import type { FeatureConfig } from '@/types/common';
 import TeraStratifiedMatrixModal from './model-configurations/tera-stratified-matrix-modal.vue';
@@ -125,8 +120,13 @@ const filterText = ref('');
 <style scoped>
 ul {
 	flex-grow: 1;
+	padding-left: var(--gap-1);
+
 	li {
+		border-bottom: 1px solid var(--gray-300);
 		list-style: none;
+		margin-bottom: var(--gap-1-5);
+		padding-bottom: var(--gap-1-5);
 	}
 }
 
@@ -141,16 +141,4 @@ ul {
 	margin-left: var(--gap-1);
 	margin-right: auto;
 }
-
-:deep(.p-divider) {
-	&.p-divider-horizontal {
-		margin-top: var(--gap-2);
-		margin-bottom: var(--gap-2);
-		color: var(--gray-300);
-	}
-	&.p-divider-vertical {
-		margin-left: var(--gap-small);
-		margin-right: var(--gap);
-	}
-}
 </style>
diff --git a/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-config-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-config-drilldown.vue
index 55405d5d6a..536def7d58 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-config-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-config-drilldown.vue
@@ -14,22 +14,17 @@
 				content-width="360px"
 			>
 				<template #content>
-					<div class="m-3">
+					<main class="m-3">
 						<div class="flex flex-row gap-1">
 							<tera-input-text v-model="filterModelConfigurationsText" placeholder="Filter" class="w-full" />
-
 							<Button
 								label="Extract from inputs"
-								@click="extractConfigurationsFromInputs"
 								severity="primary"
 								class="white-space-nowrap min-w-min"
+								size="small"
 								:loading="isLoading"
-								:disabled="
-									!props.node.inputs[1]?.value &&
-									!props.node.inputs[2]?.value &&
-									!props.node.inputs[3]?.value &&
-									!props.node.inputs[4]?.value
-								"
+								:disabled="!props.node.inputs.slice(1, 5).some((input) => input?.value)"
+								@click="extractConfigurationsFromInputs"
 							/>
 						</div>
 						<!-- Show a spinner if loading -->
@@ -42,21 +37,20 @@
 							<li v-for="configuration in filteredModelConfigurations" :key="configuration.id">
 								<tera-model-configuration-item
 									:configuration="configuration"
-									@click="onSelectConfiguration(configuration)"
 									:selected="selectedConfigId === configuration.id"
-									@use="onSelectConfiguration(configuration)"
+									@click="onSelectConfiguration(configuration)"
 									@delete="fetchConfigurations(model.id)"
 									@download="downloadModelArchive(configuration)"
+									@use="onSelectConfiguration(configuration)"
 								/>
 							</li>
 							<!-- Show a message if nothing found after filtering -->
 							<li v-if="filteredModelConfigurations.length === 0">No configurations found</li>
 						</ul>
-					</div>
+					</main>
 				</template>
 			</tera-slider-panel>
 		</template>
-
 		<tera-drilldown-section :tabName="ConfigTabs.Wizard" class="px-3 mb-10">
 			<template #header-controls-left>
 				<tera-toggleable-input
@@ -66,9 +60,15 @@
 				/>
 			</template>
 			<template #header-controls-right>
-				<Button label="Reset" @click="resetConfiguration" outlined severity="secondary" />
-				<Button label="Save as..." outlined severity="secondary" @click="showSaveModal = true" />
-				<Button class="mr-2" :disabled="isSaveDisabled" label="Save" @click="onSaveConfiguration" />
+				<Button
+					label="Reset"
+					outlined
+					severity="secondary"
+					:disabled="!isModelConfigChanged"
+					@click="resetConfiguration"
+				/>
+				<Button label="Save as" outlined severity="secondary" @click="showSaveModal = true" />
+				<Button :disabled="isSaveDisabled" label="Save" @click="onSaveConfiguration" />
 			</template>
 			<Accordion multiple :active-index="[0, 1]">
 				<AccordionTab>
@@ -178,6 +178,7 @@
 			</tera-drilldown-preview>
 		</tera-columnar-panel>
 	</tera-drilldown>
+
 	<tera-save-asset-modal
 		:initial-name="knobs.transientModelConfig.name"
 		:is-visible="showSaveModal"
@@ -186,6 +187,7 @@
 		@close-modal="showSaveModal = false"
 		@on-save="onSaveAsModelConfiguration"
 	/>
+
 	<!-- Matrix effect easter egg  -->
 	<canvas id="matrix-canvas" />
 </template>
@@ -286,6 +288,11 @@ const calibratedConfigObservables = computed<Observable[]>(() =>
 	}))
 );
 
+// Check if the model configuration is the same as the original
+const isModelConfigChanged = computed(
+	() => !isModelConfigsEqual(originalConfig.value, knobs.value.transientModelConfig)
+);
+
 // Save button is disabled if the model configuration name is empty, the values have changed, or the configuration is the same as the original
 const isSaveDisabled = computed(
 	() =>
@@ -627,7 +634,6 @@ const applyConfigValues = (config: ModelConfiguration) => {
 			state: omit(state, ['transientModelConfig'])
 		});
 	}
-	logger.success(`Configuration applied ${config.name}`);
 };
 
 const onEditDescription = async () => {
@@ -794,9 +800,29 @@ onUnmounted(() => {
 	padding: var(--gap-2);
 }
 
-ul {
-	list-style: none;
-	padding-top: var(--gap-small);
+.input-config {
+	ul {
+		list-style: none;
+		padding-top: var(--gap-4);
+	}
+
+	li {
+		& > * {
+			border-bottom: 1px solid var(--gray-300);
+			border-right: 1px solid var(--gray-300);
+		}
+
+		&:first-child > * {
+			border-top: 1px solid var(--gray-300);
+			border-top-left-radius: var(--border-radius);
+			border-top-right-radius: var(--border-radius);
+		}
+
+		&:last-child > * {
+			border-bottom-left-radius: var(--border-radius);
+			border-bottom-right-radius: var(--border-radius);
+		}
+	}
 }
 
 button.start-edit {
diff --git a/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-configuration-item.vue b/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-configuration-item.vue
index 99659c4e72..ffb0b719bb 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-configuration-item.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/model-config/tera-model-configuration-item.vue
@@ -1,15 +1,12 @@
 <template>
-	<div class="config-card" :class="{ 'card-selected': selected }">
-		<div>
-			<span class="flex align-items-center"
-				><h6>{{ configuration.name }}</h6>
-				<Button class="ml-auto" text icon="pi pi-ellipsis-v" @click.stop="toggleContextMenu"></Button
-			></span>
-			<ContextMenu ref="contextMenu" :model="contextMenuItems"></ContextMenu>
-			<p>{{ configuration.description }}</p>
-			<p>{{ formatTimestamp(configuration.createdOn) }}</p>
-		</div>
-		<Divider />
+	<div :class="{ selected: selected }">
+		<header>
+			<h6>{{ configuration.name }}</h6>
+			<Button text icon="pi pi-ellipsis-v" @click.stop="toggleContextMenu" />
+		</header>
+		<ContextMenu ref="contextMenu" :model="contextMenuItems" />
+		<p>{{ configuration.description }}</p>
+		<p>{{ formatTimestamp(configuration.createdOn) }}</p>
 	</div>
 </template>
 
@@ -17,7 +14,6 @@
 import { ModelConfiguration } from '@/types/Types';
 import { formatTimestamp } from '@/utils/date';
 import Button from 'primevue/button';
-import Divider from 'primevue/divider';
 import ContextMenu from 'primevue/contextmenu';
 import { ref } from 'vue';
 import { useConfirm } from 'primevue/useconfirm';
@@ -77,30 +73,42 @@ const onDeleteConfiguration = () => {
 </script>
 
 <style scoped>
-.config-card {
+div {
 	background: var(--surface-0);
 	border-left: 4px solid var(--surface-300);
+	padding: var(--gap-1) var(--gap-1) var(--gap-3) var(--gap-3);
+
 	&:hover {
-		cursor: pointer;
 		background-color: var(--gray-50);
+		cursor: pointer;
+	}
+
+	&.selected {
+		border-left-color: var(--primary-color);
 	}
-	&.card-selected {
-		border-left: 4px solid var(--primary-color);
+
+	&,
+	&.selected {
+		transition: border-left-color 250ms;
 	}
+}
+
+header {
+	display: flex;
+	align-items: center;
 
-	> div {
-		padding-left: var(--gap);
+	& > *:last-child {
+		margin-left: auto;
 	}
 }
 
 p {
 	color: var(--text-color-subdued);
+	font-size: var(--font-caption);
+	padding-right: var(--gap-6);
 }
-:deep(.p-divider) {
-	&.p-divider-horizontal {
-		margin-top: var(--gap);
-		margin-bottom: 0;
-		color: var(--gray-300);
-	}
+
+p + p {
+	margin-top: var(--gap-2);
 }
 </style>
diff --git a/packages/gollm/gollm_openai/prompts/config_from_document.py b/packages/gollm/gollm_openai/prompts/config_from_document.py
index 10e4450dea..6ab4a72378 100644
--- a/packages/gollm/gollm_openai/prompts/config_from_document.py
+++ b/packages/gollm/gollm_openai/prompts/config_from_document.py
@@ -26,13 +26,13 @@
     a.	`target` should reference the id of the initial variable from the AMR model ODE semantics.
     b.	`source` should reference the title or file name of the research paper.
     c.	`type` should be set to "initial”.
-    d.	You should extract a numerical value or an expression of the initial state from the user-provided text if possible and add it to `expression`
+    d.	You should extract a numerical value or an expression of the initial state from the user-provided text if possible and add it to `expression`. If you can not find a value in the text leave expression and expressionMathml blank.
     e.	`expression_mathml` should be `expression` written in MathML format.
 5.	For each parameter specified in the AMR model ODE semantics, create a parameter semantic object. Do not create new parameter semantic objects if they are not included in the original AMR model. You should set parameter semantic object fields using the following rules:
     a.	`reference_id` should reference the id of the parameter.
     b.	`source` should reference the title or file name of the research paper.
     c.	`type` should be set to "parameter".
-    d.	Be sure to extract parameter values from the user-provided text, and do not use the default values from the AMR model.
+    d.	Be sure to extract parameter values from the user-provided text, and do not use the default values from the AMR model. If you can not find a value in the text set `type` to "Constant" and leave `value`, `minimum`, and `maximum` blank.
         -	If the extracted parameter value is a single constant value, set the parameter `value` to the constant value and set `type` to "Constant".
         -	If the extracted parameter value is a distribution with a maximum and minimum value, set `type` to only "Uniform" and populate the `minimum` and `maximum` fields.
 6. `observableSemanticList` should be an empty list.

From 79f8b03103514924794bb90c5d933449159a228a Mon Sep 17 00:00:00 2001
From: Yohann Paris <github@yohannparis.com>
Date: Wed, 9 Oct 2024 13:20:30 -0400
Subject: [PATCH 6/6] 5038 bug intervention policies are not described well in
 simulate (#5079)

---
 .../tera-intervention-summary-card.vue        | 57 +++++++++++++++++++
 .../tera-calibrate-ciemss-drilldown.vue       |  2 +-
 .../tera-intervention-policy-card.vue         | 54 ++++++++++--------
 .../tera-intervention-policy-drilldown.vue    | 24 +++++++-
 .../tera-intervention-summary-card.vue        | 49 ----------------
 .../tera-simulate-ciemss-drilldown.vue        |  2 +-
 6 files changed, 110 insertions(+), 78 deletions(-)
 create mode 100644 packages/client/hmi-client/src/components/intervention-policy/tera-intervention-summary-card.vue
 delete mode 100644 packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-intervention-summary-card.vue

diff --git a/packages/client/hmi-client/src/components/intervention-policy/tera-intervention-summary-card.vue b/packages/client/hmi-client/src/components/intervention-policy/tera-intervention-summary-card.vue
new file mode 100644
index 0000000000..9de1c7a679
--- /dev/null
+++ b/packages/client/hmi-client/src/components/intervention-policy/tera-intervention-summary-card.vue
@@ -0,0 +1,57 @@
+<template>
+	<div>
+		<aside>{{ !isEmpty(intervention.dynamicInterventions) ? 'Dynamic' : 'Static' }}</aside>
+		<h5>{{ intervention.name }}</h5>
+		<ul>
+			<li v-for="(staticIntervention, index) in intervention.staticInterventions" :key="`static-${index}`">
+				Set {{ staticIntervention.type }} <strong>{{ staticIntervention.appliedTo }}</strong> to
+				<strong>{{ staticIntervention.value }}</strong> starting at <strong>{{ staticIntervention.timestep }}</strong
+				>&nbsp;day.
+			</li>
+			<li v-for="(dynamicIntervention, index) in intervention.dynamicInterventions" :key="`dynamic-${index}`">
+				Set {{ dynamicIntervention.type }} <strong>{{ dynamicIntervention.appliedTo }}</strong> to
+				<strong>{{ dynamicIntervention.value }}</strong> when
+				<strong>{{ dynamicIntervention.parameter }}</strong> crosses the threshold&nbsp;<strong>{{
+					dynamicIntervention.threshold
+				}}</strong
+				>.
+			</li>
+		</ul>
+	</div>
+</template>
+
+<script setup lang="ts">
+import { Intervention } from '@/types/Types';
+import { isEmpty } from 'lodash';
+
+defineProps<{
+	intervention: Intervention;
+}>();
+</script>
+
+<style scoped>
+div {
+	background: var(--surface-section);
+	border: 1px solid var(--surface-border-light);
+	border-radius: var(--border-radius-medium);
+	padding: var(--gap-4) var(--gap-5);
+	box-shadow:
+		0 2px 2px -1px rgba(0, 0, 0, 0.06),
+		0 2px 4px -1px rgba(0, 0, 0, 0.08);
+}
+
+ul {
+	list-style: none;
+	margin-top: var(--gap-2);
+}
+
+li + li {
+	margin-top: var(--gap-1);
+}
+
+aside {
+	color: var(--text-color-subdued);
+	font-size: var(--font-caption);
+	float: right;
+}
+</style>
diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index 69ef6bfede..fe1753502f 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -498,7 +498,7 @@ import TeraSaveSimulationModal from '@/components/project/tera-save-simulation-m
 import { useClientEvent } from '@/composables/useClientEvent';
 import { useDrilldownChartSize } from '@/composables/useDrilldownChartSize';
 import { flattenInterventionData, getInterventionPolicyById } from '@/services/intervention-policy';
-import TeraInterventionSummaryCard from '@/components/workflow/ops/simulate-ciemss/tera-intervention-summary-card.vue';
+import TeraInterventionSummaryCard from '@/components/intervention-policy/tera-intervention-summary-card.vue';
 import { getParameters } from '@/model-representation/service';
 import type { CalibrationOperationStateCiemss } from './calibrate-operation';
 import { renameFnGenerator, mergeResults, getErrorData } from './calibrate-utils';
diff --git a/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-card.vue b/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-card.vue
index df2bf90db7..61032a08a5 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-card.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-card.vue
@@ -1,15 +1,12 @@
 <template>
-	<div class="policy-card" :class="{ 'card-selected': selected }">
-		<div>
-			<span class="flex align-items-center">
-				<h6>{{ interventionPolicy.name }}</h6>
-				<Button class="ml-auto" text icon="pi pi-ellipsis-v" @click.stop="toggleContextMenu" />
-			</span>
-			<ContextMenu ref="contextMenu" :model="contextMenuItems"></ContextMenu>
-			<p>{{ interventionPolicy.description }}</p>
-			<p>{{ formatTimestamp(interventionPolicy.createdOn) }}</p>
-		</div>
-		<Divider />
+	<div :class="{ selected: selected }">
+		<header>
+			<h6>{{ interventionPolicy.name }}</h6>
+			<Button text icon="pi pi-ellipsis-v" @click.stop="toggleContextMenu" />
+		</header>
+		<ContextMenu ref="contextMenu" :model="contextMenuItems"></ContextMenu>
+		<p v-if="interventionPolicy.description">{{ interventionPolicy.description }}</p>
+		<p>{{ formatTimestamp(interventionPolicy.createdOn) }}</p>
 	</div>
 </template>
 
@@ -17,7 +14,6 @@
 import { formatTimestamp } from '@/utils/date';
 import ContextMenu from 'primevue/contextmenu';
 import { ref } from 'vue';
-import Divider from 'primevue/divider';
 import Button from 'primevue/button';
 import { InterventionPolicy } from '@/types/Types';
 
@@ -45,32 +41,42 @@ const toggleContextMenu = (event) => {
 </script>
 
 <style scoped>
-.policy-card {
+div {
 	background-color: var(--gray-0);
 	border-left: 4px solid var(--surface-300);
+	border-bottom: 1px solid var(--gray-300);
+	padding: var(--gap-1) var(--gap-1) var(--gap-3) var(--gap-3);
 
-	&.card-selected {
-		border-left: 4px solid var(--primary-color);
+	&.selected {
+		border-left-color: var(--primary-color);
+	}
+
+	&,
+	&.selected {
+		transition: border-left-color 250ms;
 	}
 
 	&:not(.card-selected):hover {
-		cursor: pointer;
 		background-color: var(--gray-50);
+		cursor: pointer;
 	}
+}
+
+header {
+	align-items: center;
+	display: flex;
 
-	& > div {
-		padding-left: var(--gap-small);
+	& > *:last-child {
+		margin-left: auto;
 	}
 }
 
 p {
 	color: var(--text-color-subdued);
+	font-size: var(--font-caption);
 }
-:deep(.p-divider) {
-	&.p-divider-horizontal {
-		margin-top: var(--gap);
-		margin-bottom: 0;
-		color: var(--gray-300);
-	}
+
+p + p {
+	margin-top: var(--gap-2);
 }
 </style>
diff --git a/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-drilldown.vue
index 87b2d1d3de..c4f8f0fd0a 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/intervention-policy/tera-intervention-policy-drilldown.vue
@@ -45,7 +45,7 @@
 			</tera-slider-panel>
 		</template>
 		<tera-columnar-panel>
-			<tera-drilldown-section class="px-3 intervention-settings-section">
+			<tera-drilldown-section class="intervention-settings-section">
 				<template #header-controls-left> Add and configure intervention settings for this policy. </template>
 				<template #header-controls-right>
 					<Button outlined severity="secondary" label="Reset" @click="onResetPolicy" />
@@ -556,10 +556,28 @@ onMounted(() => {
 <style scoped>
 .intervention-settings-section {
 	background-color: var(--surface-100);
+	padding: 0 var(--gap-3);
 }
 
-ul {
-	list-style: none;
+.input-config {
+	ul {
+		list-style: none;
+	}
+	li {
+		& > * {
+			border-bottom: 1px solid var(--gray-300);
+			border-right: 1px solid var(--gray-300);
+		}
+		&:first-child > * {
+			border-top: 1px solid var(--gray-300);
+			border-top-left-radius: var(--border-radius);
+			border-top-right-radius: var(--border-radius);
+		}
+		&:last-child > * {
+			border-bottom-left-radius: var(--border-radius);
+			border-bottom-right-radius: var(--border-radius);
+		}
+	}
 }
 
 section {
diff --git a/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-intervention-summary-card.vue b/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-intervention-summary-card.vue
deleted file mode 100644
index 39eecab607..0000000000
--- a/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-intervention-summary-card.vue
+++ /dev/null
@@ -1,49 +0,0 @@
-<template>
-	<section class="policy-group">
-		<h5>{{ intervention.name }}</h5>
-		<ul>
-			<li v-for="(staticIntervention, index) in intervention.staticInterventions" :key="index">
-				Set the <strong>{{ staticIntervention.type }}</strong
-				>&nbsp; <strong>{{ staticIntervention.appliedTo }}</strong> to the value of
-				<strong>{{ staticIntervention.value }}</strong> day at start time
-				<strong>{{ staticIntervention.timestep }}</strong> day.
-			</li>
-			<li v-for="(dynamicIntervention, index) in intervention.dynamicInterventions" :key="`dynamic_${index}`">
-				Set the {{ dynamicIntervention.type }}&nbsp; <strong>{{ dynamicIntervention.appliedTo }}</strong> to
-				<strong>{{ dynamicIntervention.threshold }}</strong> days when it crosses the threshold value
-				<strong>{{ dynamicIntervention.value }}</strong> person.
-			</li>
-		</ul>
-	</section>
-</template>
-
-<script setup lang="ts">
-import { Intervention } from '@/types/Types';
-
-defineProps<{
-	intervention: Intervention;
-}>();
-</script>
-
-<style scoped>
-.policy-group {
-	display: flex;
-	padding: var(--gap-4);
-	padding-left: var(--gap-5);
-	flex-direction: column;
-	justify-content: center;
-	align-items: flex-start;
-	gap: var(--gap-2);
-	border-radius: var(--gap-1-5);
-	background: var(--surface-section);
-	border: 1px solid var(--surface-border-light);
-	/* Shadow/medium */
-	box-shadow:
-		0 2px 4px -1px rgba(0, 0, 0, 0.06),
-		0 4px 6px -1px rgba(0, 0, 0, 0.08);
-}
-
-ul {
-	list-style: none;
-}
-</style>
diff --git a/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-simulate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-simulate-ciemss-drilldown.vue
index 155776b02c..ce60fbe289 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-simulate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/simulate-ciemss/tera-simulate-ciemss-drilldown.vue
@@ -242,7 +242,7 @@ import VegaChart from '@/components/widgets/VegaChart.vue';
 import { CiemssPresetTypes, DrilldownTabs } from '@/types/common';
 import { getModelConfigurationById } from '@/services/model-configurations';
 import { flattenInterventionData, getInterventionPolicyById } from '@/services/intervention-policy';
-import TeraInterventionSummaryCard from '@/components/workflow/ops/simulate-ciemss/tera-intervention-summary-card.vue';
+import TeraInterventionSummaryCard from '@/components/intervention-policy/tera-intervention-summary-card.vue';
 import TeraSaveSimulationModal from '@/components/project/tera-save-simulation-modal.vue';
 import { SimulateCiemssOperationState } from './simulate-ciemss-operation';
 import TeraChartControl from '../../tera-chart-control.vue';