test: test exported code executes

Resolves #8

tests/execution_utils.py

@@ -0,0 +1,41 @@
+import base64
+import pickle
+
+import nbconvert
+import nbformat
+import pandas as pd
+
+from edvart.report import ReportBase
+from edvart.report_sections.code_string_formatting import code_dedent
+from edvart.report_sections.section_base import Section
+
+
+def check_section_executes(section: Section, df: pd.DataFrame) -> None:
+    nb = nbformat.v4.new_notebook()
+    section_code_cells = []
+    section.add_cells(section_code_cells, df)
+
+    buffer = pickle.dumps(df, fix_imports=False)
+    buffer_base64 = base64.b85encode(buffer)
+
+    unpickle_df = code_dedent(
+        f"""
+        import pickle
+        import base64
+
+        data = {buffer_base64}
+        df = pickle.loads(base64.b85decode(data), fix_imports=False)"""
+    )
+
+    all_imports = [
+        *ReportBase._DEFAULT_IMPORTS,
+        *section.required_imports(),
+    ]
+
+    nb["cells"] = [
+        nbformat.v4.new_code_cell("\n".join(all_imports)),
+        nbformat.v4.new_code_cell(unpickle_df),
+        *section_code_cells,
+    ]
+    preprocessor = nbconvert.preprocessors.ExecutePreprocessor(timeout=60)
+    preprocessor.preprocess(nb)

tests/test_bivariate_analysis.py

@@ -1,6 +1,7 @@
 import warnings
 from contextlib import redirect_stdout
 
+import numpy as np
 import pandas as pd
 import pytest
 
@@ -9,6 +10,8 @@
 from edvart.report_sections.code_string_formatting import get_code
 from edvart.report_sections.section_base import Verbosity
 
+from .execution_utils import check_section_executes
+
 
 def get_test_df() -> pd.DataFrame:
     test_df = pd.DataFrame(data=[[1.1, "a"], [2.2, "b"], [3.3, "c"]], columns=["A", "B"])
@@ -131,6 +134,7 @@ def test_code_export_verbosity_low():
     # Test code equivalence
     assert len(exported_code) == 1
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
+    check_section_executes(bivariate_section, df=get_test_df())
 
 
 def test_code_export_verbosity_low_with_subsections():
@@ -155,13 +159,26 @@ def test_code_export_verbosity_low_with_subsections():
     # Test code equivalence
     assert len(exported_code) == 1
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
+    check_section_executes(bivariate_section, df=get_test_df())
 
 
 def test_generated_code_verbosity_low_columns():
     columns = [f"col{i}" for i in range(5)]
     columns_x = [f"col_x{i}" for i in range(6)]
     columns_y = [f"col_y{i}" for i in range(4)]
     columns_pairs = [(f"first{i}", f"second{i}") for i in range(8)]
+    columns_all = (
+        columns
+        + columns_x
+        + columns_y
+        + [col_pair[0] for col_pair in columns_pairs]
+        + [col_pair[1] for col_pair in columns_pairs]
+    )
+    test_df = pd.DataFrame(
+        data=np.random.rand(4, len(columns_all)),
+        columns=columns_all,
+    )
+
     bivariate_section = bivariate_analysis.BivariateAnalysis(
         columns=columns,
         columns_x=columns_x,
@@ -184,6 +201,8 @@ def test_generated_code_verbosity_low_columns():
     assert len(exported_code) == 1
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
 
+    check_section_executes(bivariate_section, df=test_df)
+
 
 def test_generated_code_verbosity_medium():
     bivariate_section = bivariate_analysis.BivariateAnalysis(
@@ -209,6 +228,8 @@ def test_generated_code_verbosity_medium():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(bivariate_section, df=get_test_df())
+
 
 def test_generated_code_verbosity_medium_columns_x_y():
     columns_x = ["a", "b"]
@@ -224,6 +245,10 @@ def test_generated_code_verbosity_medium_columns_x_y():
         ],
         color_col="b",
     )
+    test_df = pd.DataFrame(
+        columns=columns_x + columns_y,
+        data=np.random.rand(10, 4),
+    )
 
     exported_cells = []
     bivariate_section.add_cells(exported_cells, df=pd.DataFrame())
@@ -239,6 +264,8 @@ def test_generated_code_verbosity_medium_columns_x_y():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(bivariate_section, df=test_df)
+
 
 def test_generated_code_verbosity_medium_columns_pairs():
     columns_pairs = [("a", "b"), ("c", "d")]
@@ -253,6 +280,10 @@ def test_generated_code_verbosity_medium_columns_pairs():
             BivariateAnalysisSubsection.ContingencyTable,
         ],
     )
+    test_df = pd.DataFrame(
+        columns=columns_x_correct + columns_y_correct,
+        data=np.random.rand(10, 4),
+    )
 
     exported_cells = []
     bivariate_section.add_cells(exported_cells, df=pd.DataFrame())
@@ -268,6 +299,8 @@ def test_generated_code_verbosity_medium_columns_pairs():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(bivariate_section, df=test_df)
+
 
 def test_generated_code_verbosity_high():
     bivariate_section = bivariate_analysis.BivariateAnalysis(
@@ -307,6 +340,8 @@ def test_generated_code_verbosity_high():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(bivariate_section, df=get_test_df())
+
 
 def test_verbosity_low_different_subsection_verbosities():
     bivariate_section = BivariateAnalysis(
@@ -337,6 +372,8 @@ def test_verbosity_low_different_subsection_verbosities():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(bivariate_section, df=get_test_df())
+
 
 def test_imports_verbosity_low():
     bivariate_section = BivariateAnalysis(verbosity=Verbosity.LOW)

tests/test_group_analysis.py

@@ -21,6 +21,8 @@
 )
 from edvart.report_sections.section_base import Verbosity
 
+from .execution_utils import check_section_executes
+
 # Workaround to prevent multiple browser tabs opening with figures
 plotly.io.renderers.default = "json"
 
@@ -95,6 +97,8 @@ def test_code_export_verbosity_low():
     assert len(exported_code) == 1
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
 
+    check_section_executes(group_section, df)
+
 
 def test_code_export_verbosity_medium():
     df = get_test_df()
@@ -120,6 +124,7 @@ def test_code_export_verbosity_medium():
     assert len(expected_code) == len(exported_code)
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
+    check_section_executes(group_section, df)
 
 
 def test_code_export_verbosity_high():
@@ -174,6 +179,7 @@ def test_code_export_verbosity_high():
     assert len(expected_code) == len(exported_code)
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
+    check_section_executes(group_section, df)
 
 
 def test_columns_parameter():

tests/test_multivariate_analysis.py

@@ -19,6 +19,8 @@
 from edvart.report_sections.section_base import Verbosity
 from edvart.utils import select_numeric_columns
 
+from .execution_utils import check_section_executes
+
 
 def get_test_df() -> pd.DataFrame:
     test_df = pd.DataFrame(
@@ -143,6 +145,8 @@ def test_code_export_verbosity_low():
     assert len(exported_code) == 1
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
 
+    check_section_executes(multivariate_section, df)
+
 
 def test_code_export_verbosity_low_with_subsections():
     subsec = MultivariateAnalysisSubsection
@@ -181,6 +185,8 @@ def test_code_export_verbosity_low_with_subsections():
     assert len(exported_code) == 1
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
 
+    check_section_executes(multivariate_section, df)
+
 
 def test_code_export_verbosity_medium_all_cols_valid():
     all_numeric_df = pd.DataFrame(
@@ -209,6 +215,8 @@ def test_code_export_verbosity_medium_all_cols_valid():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(multivariate_section, all_numeric_df)
+
 
 def test_generated_code_verbosity_1():
     multivariate_section = MultivariateAnalysis(verbosity=Verbosity.MEDIUM)
@@ -245,6 +253,8 @@ def test_generated_code_verbosity_1():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(multivariate_section, df)
+
 
 def test_generated_code_verbosity_2():
     df = get_test_df()
@@ -306,6 +316,8 @@ def test_generated_code_verbosity_2():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(multivariate_section, df)
+
 
 def test_verbosity_medium_non_categorical_col():
     random_array = np.random.randint(low=1, high=40, size=(100, 3))
@@ -326,6 +338,8 @@ def test_verbosity_medium_non_categorical_col():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(multivariate_section, random_df)
+
 
 def test_verbosity_low_different_subsection_verbosities():
     subsections = [
@@ -370,6 +384,8 @@ def test_verbosity_low_different_subsection_verbosities():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(multivariate_section, df)
+
 
 def test_imports_verbosity_low():
     multivariate_section = MultivariateAnalysis(verbosity=Verbosity.LOW)

tests/test_overview_section.py

@@ -20,6 +20,8 @@
 from edvart.report_sections.dataset_overview import Overview, OverviewSubsection
 from edvart.report_sections.section_base import Verbosity
 
+from .execution_utils import check_section_executes
+
 
 def get_test_df() -> pd.DataFrame:
     test_df = pd.DataFrame(data=[[1.1, "a"], [2.2, "b"], [3.3, "c"]], columns=["A", "B"])
@@ -136,6 +138,8 @@ def test_code_export_verbosity_low():
     # Test code equivalence
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
 
+    check_section_executes(overview_section, df=get_test_df())
+
 
 def test_code_export_verbosity_low_with_subsections():
     overview_section = Overview(
@@ -158,6 +162,8 @@ def test_code_export_verbosity_low_with_subsections():
     # Test code equivalence
     assert exported_code[0] == expected_code[0], "Exported code mismatch"
 
+    check_section_executes(overview_section, df=get_test_df())
+
 
 def test_code_export_verbosity_medium():
     # Construct overview section
@@ -192,6 +198,8 @@ def test_code_export_verbosity_medium():
     for i in range(len(exported_code)):
         assert exported_code[i] == expected_code[i], "Exported code mismatch"
 
+    check_section_executes(overview_section, df=get_test_df())
+
 
 def test_code_export_verbosity_high():
     # Construct overview section
@@ -270,6 +278,8 @@ def test_code_export_verbosity_high():
     for i in range(len(exported_code)):
         assert exported_code[i] == expected_code[i], "Exported code mismatch"
 
+    check_section_executes(overview_section, df=get_test_df())
+
 
 def test_verbosity_low_different_subsection_verbosities():
     overview_section = Overview(
@@ -303,6 +313,8 @@ def test_verbosity_low_different_subsection_verbosities():
     for expected_line, exported_line in zip(expected_code, exported_code):
         assert expected_line == exported_line, "Exported code mismatch"
 
+    check_section_executes(overview_section, df=get_test_df())
+
 
 def test_imports_verbosity_low():
     overview_section = Overview(verbosity=Verbosity.LOW)