From 08c429abcec50f58d5fbf5109a8fb516efa4c1f8 Mon Sep 17 00:00:00 2001 From: d10n Date: Wed, 30 Oct 2024 10:03:42 -0400 Subject: [PATCH 1/7] fix: merge control markdown with json (#1528) Function cleanup / unit tests to follow Signed-off-by: d10n --- trestle/core/control_writer.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/trestle/core/control_writer.py b/trestle/core/control_writer.py index 62703bd5f..1beddc939 100644 --- a/trestle/core/control_writer.py +++ b/trestle/core/control_writer.py @@ -30,6 +30,20 @@ logger = logging.getLogger(__name__) +def comp_dict_merge(target: CompDict, source: CompDict) -> CompDict: + new_comp_dict = copy.deepcopy(target) + if not source: + return new_comp_dict + for key, val in source.items(): + if key not in new_comp_dict: + new_comp_dict[key] = val + else: + for k, v in val.items(): + if k not in new_comp_dict[key]: + new_comp_dict[key][k] = v + return new_comp_dict + + class ControlWriter(): """Class to write controls as markdown.""" @@ -518,7 +532,7 @@ def write_control_for_editing( md_header, comp_dict = ControlReader.read_control_info_from_md(control_file, context) # replace the memory comp_dict with the md one if control exists if comp_dict: - context.comp_dict = comp_dict + context.comp_dict = comp_dict_merge(context.comp_dict, comp_dict) header_comment_dict = {const.TRESTLE_ADD_PROPS_TAG: const.YAML_PROPS_COMMENT} if context.to_markdown: From fdfb3afd122d7bafcd01627320dcfed9ceb0fa1c Mon Sep 17 00:00:00 2001 From: d10n Date: Thu, 7 Nov 2024 09:34:19 -0500 Subject: [PATCH 2/7] Reuse existing merge_dicts_deep Signed-off-by: d10n --- trestle/core/control_interface.py | 2 ++ trestle/core/control_writer.py | 20 ++++---------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/trestle/core/control_interface.py b/trestle/core/control_interface.py index 1a2642d20..71fe3c456 100644 --- a/trestle/core/control_interface.py +++ b/trestle/core/control_interface.py @@ -591,6 +591,8 @@ def merge_dicts_deep( New items are always added from src to dest. Items present in both will be overriden dest if overwrite_header_values is True. """ + if src is None: + return for key in src.keys(): if key in dest: if depth and level == depth: diff --git a/trestle/core/control_writer.py b/trestle/core/control_writer.py index 1beddc939..f05b2313c 100644 --- a/trestle/core/control_writer.py +++ b/trestle/core/control_writer.py @@ -30,20 +30,6 @@ logger = logging.getLogger(__name__) -def comp_dict_merge(target: CompDict, source: CompDict) -> CompDict: - new_comp_dict = copy.deepcopy(target) - if not source: - return new_comp_dict - for key, val in source.items(): - if key not in new_comp_dict: - new_comp_dict[key] = val - else: - for k, v in val.items(): - if k not in new_comp_dict[key]: - new_comp_dict[key][k] = v - return new_comp_dict - - class ControlWriter(): """Class to write controls as markdown.""" @@ -530,9 +516,11 @@ def write_control_for_editing( control_file = dest_path / (control.id + const.MARKDOWN_FILE_EXT) # read the existing markdown header and content if it exists md_header, comp_dict = ControlReader.read_control_info_from_md(control_file, context) - # replace the memory comp_dict with the md one if control exists + # Merge the memory comp_dict with the md one if control exists if comp_dict: - context.comp_dict = comp_dict_merge(context.comp_dict, comp_dict) + template_comp_dict = context.comp_dict + ControlInterface.merge_dicts_deep(comp_dict, template_comp_dict, False) + context.comp_dict = comp_dict header_comment_dict = {const.TRESTLE_ADD_PROPS_TAG: const.YAML_PROPS_COMMENT} if context.to_markdown: From 8f87b7dfd61ae1da19622dc4da49e700361eab9d Mon Sep 17 00:00:00 2001 From: d10n Date: Thu, 7 Nov 2024 09:40:05 -0500 Subject: [PATCH 3/7] Enable unit tests to run in PyCharm PyCharm has trouble keeping the working directory consistent, so make the tests look for files relative to the test file instead of the CWD Signed-off-by: d10n --- tests/conftest.py | 4 +++- tests/test_utils.py | 17 +++++++++-------- tests/trestle/core/commands/validate_test.py | 18 ++++++++++++------ tests/trestle/core/control_io_test.py | 7 ++++--- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 21704a0f8..5746c44af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -259,7 +259,9 @@ def tmp_empty_cwd(tmp_path: pathlib.Path) -> Iterator[pathlib.Path]: @pytest.fixture(scope='function') def testdata_dir() -> pathlib.Path: """Return absolute path to test data directory.""" - test_data_source = pathlib.Path('tests/data') + test_dir = pathlib.Path(__file__).parent.resolve() + data_path = test_dir / 'data' + test_data_source = pathlib.Path(data_path) return test_data_source.resolve() diff --git a/tests/test_utils.py b/tests/test_utils.py index 29c746ade..976a730b7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -53,26 +53,27 @@ logger = logging.getLogger(__name__) -BASE_TMP_DIR = pathlib.Path('tests/__tmp_path').resolve() -YAML_TEST_DATA_PATH = pathlib.Path('tests/data/yaml/').resolve() -JSON_TEST_DATA_PATH = pathlib.Path('tests/data/json/').resolve() -ENV_TEST_DATA_PATH = pathlib.Path('tests/data/env/').resolve() -JSON_NIST_DATA_PATH = pathlib.Path('nist-content/nist.gov/SP800-53/rev5/json/').resolve() +TEST_DIR = pathlib.Path(__file__).parent.resolve() +BASE_TMP_DIR = pathlib.Path(TEST_DIR / '__tmp_path').resolve() +YAML_TEST_DATA_PATH = pathlib.Path(TEST_DIR / 'data/yaml/').resolve() +JSON_TEST_DATA_PATH = pathlib.Path(TEST_DIR / 'data/json/').resolve() +ENV_TEST_DATA_PATH = pathlib.Path(TEST_DIR / 'data/env/').resolve() +JSON_NIST_DATA_PATH = pathlib.Path(TEST_DIR / '../nist-content/nist.gov/SP800-53/rev5/json/').resolve() JSON_NIST_CATALOG_NAME = 'NIST_SP-800-53_rev5_catalog.json' JSON_NIST_PROFILE_NAME = 'NIST_SP-800-53_rev5_MODERATE-baseline_profile.json' -JSON_NIST_REV_4_DATA_PATH = pathlib.Path('nist-content/nist.gov/SP800-53/rev4/json/').resolve() +JSON_NIST_REV_4_DATA_PATH = pathlib.Path(TEST_DIR / '../nist-content/nist.gov/SP800-53/rev4/json/').resolve() JSON_NIST_REV_4_CATALOG_NAME = 'NIST_SP-800-53_rev4_catalog.json' JSON_NIST_REV_5_CATALOG_NAME = 'nist-rev5-catalog-full.json' JSON_NIST_REV_4_PROFILE_NAME = 'NIST_SP-800-53_rev4_MODERATE-baseline_profile.json' SIMPLIFIED_NIST_CATALOG_NAME = 'simplified_nist_catalog.json' SIMPLIFIED_NIST_PROFILE_NAME = 'simplified_nist_profile.json' -TASK_XLSX_OUTPUT_PATH = pathlib.Path('tests/data/tasks/xlsx/output').resolve() +TASK_XLSX_OUTPUT_PATH = pathlib.Path(TEST_DIR / 'data/tasks/xlsx/output').resolve() CATALOGS_DIR = 'catalogs' PROFILES_DIR = 'profiles' COMPONENT_DEF_DIR = 'component-definitions' -NIST_EXAMPLES = pathlib.Path('nist-content/examples') +NIST_EXAMPLES = pathlib.Path(TEST_DIR / '../nist-content/examples') NIST_SAMPLE_CD_JSON = NIST_EXAMPLES / 'component-definition' / 'json' / 'example-component.json' NEW_MODEL_AGE_SECONDS = 100 diff --git a/tests/trestle/core/commands/validate_test.py b/tests/trestle/core/commands/validate_test.py index 8eb02f457..3b9c0d5a9 100644 --- a/tests/trestle/core/commands/validate_test.py +++ b/tests/trestle/core/commands/validate_test.py @@ -46,7 +46,6 @@ from trestle.oscal.common import ResponsibleParty, Role from trestle.oscal.component import ComponentDefinition, ControlImplementation -test_data_dir = pathlib.Path('tests/data').resolve() md_path = 'md_comp' @@ -58,7 +57,8 @@ ('my_test_model', '-t', False), ('my_test_model', '-a', False), ('my_test_model', '-x', False) ] ) -def test_validation_happy(name, mode, parent, tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validation_happy(name, mode, parent, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: + test_data_dir = testdata_dir """Test successful validation runs.""" (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True) (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2').mkdir(exist_ok=True, parents=True) @@ -101,8 +101,9 @@ def test_validation_happy(name, mode, parent, tmp_trestle_dir: pathlib.Path, mon ] ) def test_validation_unhappy( - name, mode, parent, status, tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch + name, mode, parent, status, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch ) -> None: + test_data_dir = testdata_dir """Test failure modes of validation.""" (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True) (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2').mkdir(exist_ok=True, parents=True) @@ -421,8 +422,10 @@ def test_period(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None pass -def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: """Test validation of Component Definition.""" + test_data_dir = testdata_dir + jfile = 'component-definition.json' sdir = test_data_dir / 'validate' / 'component-definitions' / 'x1' @@ -438,8 +441,9 @@ def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, monkeypatc test_utils.execute_command_and_assert(validate_command, 0, monkeypatch) -def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: """Test validation of ports in Component Definition.""" + test_data_dir = testdata_dir jfile = 'component-definition.json' sdir = test_data_dir / 'validate' / 'component-definitions' / 'x2' @@ -455,8 +459,10 @@ def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, monk test_utils.execute_command_and_assert(validate_command, 0, monkeypatch) -def test_validate_component_definition_ports_invalid(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validate_component_definition_ports_invalid(tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: """Test validation of ports in Component Definition.""" + test_data_dir = testdata_dir + jfile = 'component-definition.json' sdir = test_data_dir / 'validate' / 'component-definitions' / 'x3' diff --git a/tests/trestle/core/control_io_test.py b/tests/trestle/core/control_io_test.py index 1ecf96985..c83cc9dcd 100644 --- a/tests/trestle/core/control_io_test.py +++ b/tests/trestle/core/control_io_test.py @@ -30,7 +30,7 @@ from trestle.common.model_utils import ModelUtils from trestle.core.catalog.catalog_interface import CatalogInterface from trestle.core.control_context import ContextPurpose, ControlContext -from trestle.core.control_interface import ControlInterface, ParameterRep +from trestle.core.control_interface import ControlInterface, ParameterRep, ComponentImpInfo from trestle.core.control_reader import ControlReader from trestle.core.control_writer import ControlWriter from trestle.core.markdown.control_markdown_node import ControlMarkdownNode, tree_context @@ -39,6 +39,7 @@ from trestle.core.models.file_content_type import FileContentType from trestle.core.profile_resolver import ProfileResolver from trestle.oscal import common +from trestle.oscal.common import ImplementationStatus case_1 = 'indent_normal' case_2 = 'indent jump back 2' @@ -338,10 +339,10 @@ def test_get_control_param_dict(tmp_trestle_dir: pathlib.Path) -> None: @pytest.mark.parametrize('overwrite_header_values', [True, False]) -def test_write_control_header_params(overwrite_header_values, tmp_path: pathlib.Path) -> None: +def test_write_control_header_params(overwrite_header_values, tmp_path: pathlib.Path, testdata_dir: pathlib.Path) -> None: """Test write/read of control header params.""" # orig file just has one param ac-1_prm_3 - src_control_path = pathlib.Path('tests/data/author/controls/control_with_components_and_params.md') + src_control_path = pathlib.Path(testdata_dir / 'author/controls/control_with_components_and_params.md') # header has two params - 3 and 4 header = { const.SET_PARAMS_TAG: { From 4003347e13ad2d2165c316715c6a6c8a87eda945 Mon Sep 17 00:00:00 2001 From: d10n Date: Thu, 7 Nov 2024 10:24:17 -0500 Subject: [PATCH 4/7] Format code Signed-off-by: d10n --- tests/trestle/core/commands/validate_test.py | 17 ++++++++++++----- tests/trestle/core/control_io_test.py | 4 +++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/trestle/core/commands/validate_test.py b/tests/trestle/core/commands/validate_test.py index 3b9c0d5a9..d180e92cc 100644 --- a/tests/trestle/core/commands/validate_test.py +++ b/tests/trestle/core/commands/validate_test.py @@ -46,7 +46,6 @@ from trestle.oscal.common import ResponsibleParty, Role from trestle.oscal.component import ComponentDefinition, ControlImplementation - md_path = 'md_comp' @@ -57,7 +56,9 @@ ('my_test_model', '-t', False), ('my_test_model', '-a', False), ('my_test_model', '-x', False) ] ) -def test_validation_happy(name, mode, parent, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validation_happy( + name, mode, parent, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch +) -> None: test_data_dir = testdata_dir """Test successful validation runs.""" (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True) @@ -422,7 +423,9 @@ def test_period(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None pass -def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validate_component_definition( + tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch +) -> None: """Test validation of Component Definition.""" test_data_dir = testdata_dir @@ -441,7 +444,9 @@ def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, testdata_d test_utils.execute_command_and_assert(validate_command, 0, monkeypatch) -def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validate_component_definition_ports( + tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch +) -> None: """Test validation of ports in Component Definition.""" test_data_dir = testdata_dir jfile = 'component-definition.json' @@ -459,7 +464,9 @@ def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, test test_utils.execute_command_and_assert(validate_command, 0, monkeypatch) -def test_validate_component_definition_ports_invalid(tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: +def test_validate_component_definition_ports_invalid( + tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch +) -> None: """Test validation of ports in Component Definition.""" test_data_dir = testdata_dir diff --git a/tests/trestle/core/control_io_test.py b/tests/trestle/core/control_io_test.py index c83cc9dcd..37ba53184 100644 --- a/tests/trestle/core/control_io_test.py +++ b/tests/trestle/core/control_io_test.py @@ -339,7 +339,9 @@ def test_get_control_param_dict(tmp_trestle_dir: pathlib.Path) -> None: @pytest.mark.parametrize('overwrite_header_values', [True, False]) -def test_write_control_header_params(overwrite_header_values, tmp_path: pathlib.Path, testdata_dir: pathlib.Path) -> None: +def test_write_control_header_params( + overwrite_header_values, tmp_path: pathlib.Path, testdata_dir: pathlib.Path +) -> None: """Test write/read of control header params.""" # orig file just has one param ac-1_prm_3 src_control_path = pathlib.Path(testdata_dir / 'author/controls/control_with_components_and_params.md') From 02b97439911f4c72f86eb155f69cc9305b9297d2 Mon Sep 17 00:00:00 2001 From: d10n Date: Tue, 12 Nov 2024 08:41:24 -0500 Subject: [PATCH 5/7] Fix docstrings Signed-off-by: d10n --- tests/trestle/core/commands/validate_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/trestle/core/commands/validate_test.py b/tests/trestle/core/commands/validate_test.py index d180e92cc..10a2c2c13 100644 --- a/tests/trestle/core/commands/validate_test.py +++ b/tests/trestle/core/commands/validate_test.py @@ -59,8 +59,8 @@ def test_validation_happy( name, mode, parent, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch ) -> None: - test_data_dir = testdata_dir """Test successful validation runs.""" + test_data_dir = testdata_dir (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True) (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2').mkdir(exist_ok=True, parents=True) shutil.copyfile( @@ -104,8 +104,8 @@ def test_validation_happy( def test_validation_unhappy( name, mode, parent, status, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch ) -> None: - test_data_dir = testdata_dir """Test failure modes of validation.""" + test_data_dir = testdata_dir (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True) (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2').mkdir(exist_ok=True, parents=True) shutil.copyfile( From 5b455c3ed5407cfa2946f737fffbaf1fcbca9468 Mon Sep 17 00:00:00 2001 From: d10n Date: Tue, 12 Nov 2024 08:46:51 -0500 Subject: [PATCH 6/7] Clean up imports Signed-off-by: d10n --- tests/trestle/core/control_io_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/trestle/core/control_io_test.py b/tests/trestle/core/control_io_test.py index 37ba53184..d93e27d6e 100644 --- a/tests/trestle/core/control_io_test.py +++ b/tests/trestle/core/control_io_test.py @@ -30,7 +30,7 @@ from trestle.common.model_utils import ModelUtils from trestle.core.catalog.catalog_interface import CatalogInterface from trestle.core.control_context import ContextPurpose, ControlContext -from trestle.core.control_interface import ControlInterface, ParameterRep, ComponentImpInfo +from trestle.core.control_interface import ControlInterface, ParameterRep from trestle.core.control_reader import ControlReader from trestle.core.control_writer import ControlWriter from trestle.core.markdown.control_markdown_node import ControlMarkdownNode, tree_context @@ -39,7 +39,6 @@ from trestle.core.models.file_content_type import FileContentType from trestle.core.profile_resolver import ProfileResolver from trestle.oscal import common -from trestle.oscal.common import ImplementationStatus case_1 = 'indent_normal' case_2 = 'indent jump back 2' From 2068811477a12e816d7784256520a8bbf74204db Mon Sep 17 00:00:00 2001 From: d10n Date: Tue, 12 Nov 2024 08:48:50 -0500 Subject: [PATCH 7/7] Fix pylint whitespace errors Signed-off-by: d10n --- trestle/core/catalog/catalog_interface.py | 2 +- trestle/transforms/implementations/tanium.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trestle/core/catalog/catalog_interface.py b/trestle/core/catalog/catalog_interface.py index 2278ddbf4..da8c5929b 100644 --- a/trestle/core/catalog/catalog_interface.py +++ b/trestle/core/catalog/catalog_interface.py @@ -904,5 +904,5 @@ def generate_control_rule_info(self, part_id_map: Dict[str, Dict[str, str]], con if len(dup_comp_uuids) > 0: # throw an exception if there are repeated component uuids for comp_uuid in dup_comp_uuids: - logger.error(f'Component uuid { comp_uuid } is duplicated') + logger.error(f'Component uuid {comp_uuid} is duplicated') raise TrestleError('Component uuids cannot be duplicated between different component definitions') diff --git a/trestle/transforms/implementations/tanium.py b/trestle/transforms/implementations/tanium.py index 33d108137..e76f21ced 100644 --- a/trestle/transforms/implementations/tanium.py +++ b/trestle/transforms/implementations/tanium.py @@ -110,7 +110,7 @@ def transform(self, blob: str) -> Results: results.__root__ = tanium_oscal_factory.results ts1 = datetime.datetime.now() self._analysis = tanium_oscal_factory.analysis - self._analysis.append(f'transform time: {ts1-ts0}') + self._analysis.append(f'transform time: {ts1 - ts0}') return results @@ -455,7 +455,7 @@ def _batch_observations(self, index: int) -> Dict[str, List[Observation]]: start = index * batch_size end = (index + 1) * batch_size end = min(end, len(self._rule_use_list)) - logger.debug(f'start: {start} end: {end-1}') + logger.debug(f'start: {start} end: {end - 1}') # process just the one chunk for i in range(start, end): rule_use = self._rule_use_list[i]