Merge pull request #21 from google/packaging

Package for PyPI

.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,38 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/dfiq
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.10' 
+    - name: Install pypa/build
+      run: >-
+        python -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        python -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+        .
+    - name: Publish distribution 📦 to PyPI
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: pypa/gh-action-pypi-publish@release/v1

README.md

@@ -10,4 +10,4 @@ consistent, thorough, and explainable investigations.
 
 * DFIQ is a catalog of investigative knowledge, centered on [Questions](https://dfiq.org/questions)
 * Uses the concept of [Scenarios](https://dfiq.org/scenarios) to logically group Questions and help structure investigations
-* Stores [data](/data) in an easily-readable, tool-agnostic format (YAML) to be used by others
+* Stores [data](/dfiq/data) in an easily-readable, tool-agnostic format (YAML) to be used by others

dfiq/__init__.py

@@ -0,0 +1,6 @@
+from .dfiq import DFIQ, Scenario, Facet, Question, Approach
+
+__all__ = ["DFIQ", "Scenario", "Facet", "Question", "Approach"]
+__version__ = "1.0.1"
+__author__ = "Ryan Benson"
+__email__ = "[email protected]"

data/facets/F1001.yaml → dfiq/data/facets/F1001.yaml

@@ -14,12 +14,12 @@
 
 ---
 display_name: >
-  Are any ExternalCompany-related files on the actor’s assigned Company
+  Are any ExternalCompany-related files on the actor's assigned Company
   assets?
 type: facet
 description: >
   An actor may bring unauthorized external intellectual property onto Company
-  “end user” devices (like laptops or desktops). Depending on the time frame
+  "end user" devices (like laptops or desktops). Depending on the time frame
   under consideration, past devices that the actor no longer actively uses may
   need to be examined as well.
 id: F1001

dfiq.py → dfiq/dfiq.py

@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import importlib.resources
 import jinja2
 import logging
 import networkx as nx
 import os
 import re
 import yamale
 import yaml
+from pathlib import Path
+
 
 logging.basicConfig(
     level=logging.DEBUG, format="%(filename)s | %(levelname)s | %(message)s"
@@ -71,9 +74,8 @@ def __init__(
         if not self.parent_ids:
             self.parent_ids = set()
 
-        if description:
-            if isinstance(description, str):
-                self.description = description.rstrip()
+        if isinstance(description, str):
+            self.description = description.rstrip()
 
         if self.id[1] == "0":
             self.is_internal = True
@@ -202,8 +204,8 @@ class DFIQ:
     relationships.
 
     Attributes:
-        yaml_data_path (str): The path to the directory containing the DFIQ YAML files.
-        markdown_output_path (str, optional): The path to the directory where the
+        yaml_data_path (Path): The path to the directory containing the DFIQ YAML files.
+        markdown_output_path (Path, optional): The path to the directory where the
             generated Markdown files should be saved.
         plural_map (dict): A dictionary mapping from DFIQ component types to their
             plural forms.
@@ -217,12 +219,14 @@ class DFIQ:
 
     def __init__(
         self,
-        yaml_data_path: str = "data",
-        markdown_output_path: str | None = None,
-        templates_path: str = "templates",
+        yaml_data_path: Path | str | None = None,
+        markdown_output_path: Path | str | None = None,
+        templates_path: Path | str | None = Path("../templates"),
     ) -> None:
         self.yaml_data_path = yaml_data_path
         self.markdown_output_path = markdown_output_path
+        if self.markdown_output_path:
+            self.markdown_output_path = Path(self.markdown_output_path)
         self.plural_map = {
             "Scenario": "scenarios",
             "Facet": "facets",
@@ -241,7 +245,9 @@ def __init__(
             "Approach": None,
         }
 
-        logging.info(f'"yaml_data_path" set to "{self.yaml_data_path}"')
+        if self.yaml_data_path:
+            self.yaml_data_path = Path(self.yaml_data_path)
+            logging.info(f'"yaml_data_path" set to "{self.yaml_data_path.resolve()}"')
 
         self._load_dfiq_schema()
         self.load_dfiq_items_from_yaml()
@@ -362,27 +368,40 @@ def load_yaml_files_by_type(
     ) -> dict:
         """Load all DFIQ YAML files of a given type from the appropriate path.
 
-        Given the yaml_data_path, locate the correct sub-directory for that
+        Given the yaml_data_path, locate the correct subdirectory for that
         dfiq_type, validate any YAML files there, and load them into a dict.
 
         Args:
             dfiq_type (str): The component type (Scenario, Facet, Question, or Approach).
             yaml_data_path (str, optional): The base path holding the YAML files.
 
         """
-        if not yaml_data_path:
-            yaml_data_path = self.yaml_data_path
         component_dict = {}
-        dfiq_files = os.listdir(
-            os.path.join(yaml_data_path, self.plural_map.get(dfiq_type))
-        )
-        for dfiq_file in dfiq_files:
-            if dfiq_file.endswith(("-template.yaml", "-blank.yaml")):
-                continue
-            file_to_open = os.path.join(
-                yaml_data_path, self.plural_map.get(dfiq_type), dfiq_file
+        yaml_file_paths = []
+
+        if not yaml_data_path:
+            dfiq_data_files = importlib.resources.files(
+                f"dfiq.data.{self.plural_map.get(dfiq_type)}"
+            )
+            for data_file_path in dfiq_data_files.iterdir():
+                # Cast the path from Traversable -> str -> Path; I could not
+                # find a more elegant way to satisfy pytype.
+                if Path(str(data_file_path)).match("*[!-]*.yaml"):
+                    yaml_file_paths.append(str(data_file_path))
+
+        if yaml_data_path:
+            dfiq_files = os.listdir(
+                os.path.join(yaml_data_path, self.plural_map.get(dfiq_type))
             )
+            for dfiq_file in dfiq_files:
+                if dfiq_file.endswith(("-template.yaml", "-blank.yaml")):
+                    continue
+                file_to_open = os.path.join(
+                    yaml_data_path, self.plural_map.get(dfiq_type), dfiq_file
+                )
+                yaml_file_paths.append(file_to_open)
 
+        for file_to_open in yaml_file_paths:
             if not self.validate_yaml_file(file_to_open):
                 continue
 
@@ -410,12 +429,32 @@ def validate_yaml_file(yaml_file_path: str) -> bool:
                 return False
             return True
 
+    @staticmethod
+    def _get_dfiq_file(subdirectory, file_name=None):
+        """Load a file bundled in the dfiq package. If multiple subdirectories are needed, use . to separate them."""
+        if file_name:
+            return importlib.resources.files(f"dfiq.{subdirectory}").joinpath(file_name)
+        else:
+            return importlib.resources.files(f"dfiq.{subdirectory}")
+
+    def _get_dfiq_directory(self, subdirectory):
+        return importlib.resources.as_file(self._get_dfiq_file(subdirectory))
+
     def _load_dfiq_schema(self) -> None:
         """Load Yamale 'spec' files to use for validation."""
-        self.schemas["Scenario"] = yamale.make_schema("utils/scenario_spec.yaml")
-        self.schemas["Facet"] = yamale.make_schema("utils/facet_spec.yaml")
-        self.schemas["Question"] = yamale.make_schema("utils/question_spec.yaml")
-        self.schemas["Approach"] = yamale.make_schema("utils/approach_spec.yaml")
+
+        self.schemas["Scenario"] = yamale.make_schema(
+            self._get_dfiq_file("utils", "scenario_spec.yaml")
+        )
+        self.schemas["Facet"] = yamale.make_schema(
+            self._get_dfiq_file("utils", "facet_spec.yaml")
+        )
+        self.schemas["Question"] = yamale.make_schema(
+            self._get_dfiq_file("utils", "question_spec.yaml")
+        )
+        self.schemas["Approach"] = yamale.make_schema(
+            self._get_dfiq_file("utils", "approach_spec.yaml")
+        )
 
     def validate_dfiq_schema(self, yaml_file_path: str, component_type: str) -> bool:
         """Validate that a YAML file adheres to the appropriate DFIQ Schema."""
@@ -470,7 +509,7 @@ def generate_scenario_md(
         scenario = self.components.get(scenario_id)
 
         if not scenario:
-            raise Exception(f"Unable to find {scenario_id} in components dictionary")
+            raise ValueError(f"Unable to find {scenario_id} in components dictionary")
 
         if scenario.is_internal and not allow_internal:
             logging.warning(
@@ -484,12 +523,12 @@ def generate_scenario_md(
             "components": self.components,
             "allow_internal": allow_internal,
         }
+
         content = template.render(context)
-        with open(
-            os.path.join(self.markdown_output_path, "scenarios", f"{scenario_id}.md"),
-            mode="w",
-        ) as file:
-            file.write(content)
+        output_path = Path(self.markdown_output_path, "scenarios", f"{scenario_id}.md")
+        output_file = Path(output_path)
+        output_file.parent.mkdir(exist_ok=True, parents=True)
+        output_file.write_text(content, encoding="utf-8")
 
     def generate_question_md(
         self,
@@ -511,7 +550,7 @@ def generate_question_md(
         question = self.components.get(question_id)
 
         if not question:
-            raise Exception(f"Unable to find {question_id} in components dictionary")
+            raise ValueError(f"Unable to find {question_id} in components dictionary")
 
         if question.is_internal and not allow_internal:
             logging.warning(
@@ -531,14 +570,19 @@ def generate_question_md(
             "components": self.components,
             "allow_internal": allow_internal,
         }
+
         content = template.render(context)
-        output_path = os.path.join(
-            self.markdown_output_path, "questions", f"{question_id}.md"
+        output_path = Path(self.markdown_output_path, "questions", f"{question_id}.md")
+        self.write_content_to_file(content, output_path)
+        logging.info(
+            f"Wrote Markdown for Question {question_id} to {output_path.resolve()}"
         )
-        with open(output_path, mode="w") as file:
-            file.write(content)
 
-        logging.info(f"Wrote Markdown for Question {question_id} to {output_path}")
+    @staticmethod
+    def write_content_to_file(content: str, file_path: Path) -> None:
+        output_file = Path(file_path)
+        output_file.parent.mkdir(exist_ok=True, parents=True)
+        output_file.write_text(content, encoding="utf-8")
 
     def generate_question_index_md(self, allow_internal: bool = False) -> None:
         """Generates Markdown for the index page listing all Questions.
@@ -553,10 +597,9 @@ def generate_question_index_md(self, allow_internal: bool = False) -> None:
         template = self.jinja_env.get_template("questions_index.jinja2")
         context = {"components": self.components, "allow_internal": allow_internal}
         content = template.render(context)
-        with open(
-            os.path.join(self.markdown_output_path, "questions", "index.md"), mode="w"
-        ) as file:
-            file.write(content)
+        output_path = Path(self.markdown_output_path, "questions", "index.md")
+        self.write_content_to_file(content, output_path)
+        logging.info(f"Wrote Markdown for Question Index to {output_path.resolve()}")
 
     def generate_approach_glossary_md(self, allow_internal: bool = False) -> None:
         """Generates Markdown for the Approach Glossary page, listing common items in Approaches.
@@ -617,10 +660,8 @@ def generate_approach_glossary_md(self, allow_internal: bool = False) -> None:
             "components": self.components,
         }
         content = template.render(context)
-        with open(
-            os.path.join(
-                self.markdown_output_path, "contributing", "approach_glossary.md"
-            ),
-            mode="w",
-        ) as file:
-            file.write(content)
+        output_path = Path(
+            self.markdown_output_path, "contributing", "approach_glossary.md"
+        )
+        self.write_content_to_file(content, output_path)
+        logging.info(f"Wrote Markdown for Approach Glossary to {output_path.resolve()}")

scripts/generate_site_markdown.py → dfiq/scripts/generate_site_markdown.py

@@ -14,7 +14,7 @@
 
 from dfiq import DFIQ
 
-dfiq_instance = DFIQ(markdown_output_path=f"site/docs")
+dfiq_instance = DFIQ(markdown_output_path="../../site/docs")
 
 for scenario in dfiq_instance.scenarios():
     dfiq_instance.generate_scenario_md(scenario.id)

pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=61", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dfiq"
+authors = [{name = "Ryan Benson", email = "[email protected]"}]
+readme = "README.md"
+requires-python = ">=3.10"
+license = {file = "LICENSE"}
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3"
+]
+dynamic = ["dependencies", "version"]
+description = "DFIQ is a collection of investigative questions and the approaches for answering them"
+keywords=["dfiq", "forensics", "dfir", "investigative questions", "security", "digital forensics"]
+
+[tool.setuptools]
+packages = [
+    "dfiq",
+    "dfiq.data.approaches",
+    "dfiq.data.facets",
+    "dfiq.data.questions",
+    "dfiq.data.scenarios",
+    "dfiq.scripts",
+    "dfiq.templates",
+    "dfiq.utils"
+]
+
+[tool.setuptools_scm]
+
+[tool.setuptools.dynamic]
+dependencies = { file = ["requirements.txt"] }
+version = { attr = "dfiq.__version__" }
+
+[project.urls]
+Homepage = "https://dfiq.org"
+Repository = "https://github.com/google/dfiq"

requirements.txt

@@ -1,4 +1,4 @@
-networkx
+networkx >= 3
 pyyaml
 jinja2
 yamale