Read data using pandas (#8)

Author: katybaulch

.env.example

@@ -16,7 +16,9 @@ permissions: read-all
 
 jobs:
   code-quality:
-    uses: climatepolicyradar/reusable-workflows/.github/workflows/python-precommit-validator-without-version.yml@v12
+    uses: climatepolicyradar/reusable-workflows/.github/workflows/python-precommit-validator-without-version.yml@v13
+    with:
+      python-version: "3.10"
 
   test:
     runs-on: ubuntu-latest
@@ -36,15 +38,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install "poetry==1.7.1"
-          poetry config virtualenvs.create false
-          poetry install --no-cache
+          poetry config virtualenvs.prefer-active-python true
+          poetry config virtualenvs.create true
           poetry install --only-root
+          poetry install --with dev --no-cache
 
       - name: Export PYTHONPATH
         run: echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV
 
-      - name: Create .env
-        run: cp .env.example .env
-
       - name: Run Tests
-        run: pytest -vvv tests/unit_tests
+        run: poetry run pytest -vvv tests/unit_tests

.github/workflows/ci-cd.yml

@@ -28,7 +28,10 @@
     "PYTHONPATH",
     "markdownlint",
     "shellcheck",
-    "SCRIPTDIR"
+    "SCRIPTDIR",
+    "chunksize",
+    "dataframe",
+    "dataframes"
   ],
   "flagWords": ["hte"],
   "suggestionsTimeout": 5000

.trunk/configs/cspell.json

@@ -43,7 +43,7 @@ lint:
         - .gitignore
     - linters: [pre-commit-hooks, prettier]
       paths:
-        - tests/unit_tests/test_fixtures/malformed_data.json
+        - tests/unit_tests/fixtures/malformed_data.json
 
   enabled:
     - [email protected]

.trunk/trunk.yaml

@@ -24,4 +24,13 @@ to find the latest version.
 
 ## Usage
 
-TBD
+If `--output_file` is not passed, by default an output file called `output.json`
+will be created in the current directory if it does not already exist.
+
+If `--gcf_projects_file`, `mcf_projects_file` or `mcf_docs_file` is not passed,
+by default the GCF mapper tool will look for a sub-folder in the current working
+directory called `data` with the following files in it:
+
+```bash
+gcf_data_mapper --gcf_projects_file FILENAME --mcf_projects_file FILENAME --mcf_docs_file FILENAME --output_file FILENAME
+```

README.md

@@ -1,25 +1,64 @@
+import os
 import sys
 from typing import Any, Optional
 
 import click
+import pandas as pd
 
 from gcf_data_mapper.parsers.collection import collection
 from gcf_data_mapper.parsers.document import document
 from gcf_data_mapper.parsers.family import family
+from gcf_data_mapper.read import read
 
 
 @click.command()
-@click.option("--debug/--no-debug", default=False)
+@click.option(
+    "--gcf_projects_file",
+    default=os.path.join(os.getcwd(), "data", "gcf-projects.json"),
+    type=click.Path(exists=True),
+)
+@click.option(
+    "--mcf_projects_file",
+    # trunk-ignore(cspell/error)
+    default=os.path.join(os.getcwd(), "data", "MCFprojects.csv"),
+    type=click.Path(exists=True),
+)
+@click.option(
+    "--mcf_docs_file",
+    # trunk-ignore(cspell/error)
+    default=os.path.join(os.getcwd(), "data", "MCFdocuments-v2.csv"),
+    type=click.Path(exists=True),
+)
+@click.option(
+    "--output_file",
+    default=os.path.join(os.getcwd(), "output.json"),
+    type=click.Path(exists=False),
+)
+@click.option("--debug/--no-debug", default=True)
 @click.version_option("0.1.0", "--version", "-v", help="Show the version and exit.")
-def entrypoint(debug: bool):
+def entrypoint(
+    gcf_projects_file, mcf_projects_file, mcf_docs_file, output_file, debug: bool
+):
     """Simple program that wrangles GCF data into bulk import format.
 
+    :param str gcf_projects_file: The GCF projects filename.
+    :param str mcf_projects_file: The MCF projects filename.
+    :param str mcf_docs_file: The MCF projects filename.
+    :param str output_file: The output filename.
     :param bool debug: Whether debug mode is on.
     """
     click.echo("🚀 Starting the GCF data mapping process.")
+    if debug:
+        click.echo("📝 Input files:")
+        click.echo(f"- {click.format_filename(gcf_projects_file)}")
+        click.echo(f"- {click.format_filename(mcf_projects_file)}")
+        click.echo(f"- {click.format_filename(mcf_docs_file)}")
 
     try:
-        wrangle_to_json(debug)
+        project_info, doc_info = read(
+            gcf_projects_file, mcf_projects_file, mcf_docs_file, debug
+        )
+        mapped_data = wrangle_to_json(project_info, doc_info, debug)
     except Exception as e:
         click.echo(f"❌ Failed to map GCF data to expected JSON. Error: {e}.")
         sys.exit(1)
@@ -28,31 +67,48 @@ def entrypoint(debug: bool):
 
     click.echo()
     click.echo("🚀 Dumping GCF data to output file")
-    dump_output()
+    dump_output(mapped_data, output_file, debug)
     click.echo("✅ Finished dumping mapped GCF data.")
 
 
-def wrangle_to_json(debug) -> dict[str, list[Optional[dict[str, Any]]]]:
+def wrangle_to_json(
+    project_info: pd.DataFrame, doc_info: pd.DataFrame, debug: bool
+) -> dict[str, list[Optional[dict[str, Any]]]]:
     """Put the mapped GCF data into a dictionary ready for dumping.
 
     The output of this function will get dumped as JSON to the output
     file.
 
+    :param pd.DataFrame project_info: The GCF and MCF joined project
+        info.
+    :param pd.DataFrame doc_info: The MCF docs info.
     :param bool debug: Whether debug mode is on.
     :return dict[str, list[Optional[dict[str, Any]]]]: The GCF data
         mapped to the Document-Family-Collection-Event entity it
         corresponds to.
     """
     return {
         "collections": collection(debug),
-        "families": family(debug),
-        "documents": document(debug),
+        "families": family(project_info, debug),
+        "documents": document(doc_info, debug),
         "events": [],
     }
 
 
-def dump_output():
-    pass
+def dump_output(
+    mapped_data: dict[str, list[Optional[dict[str, Any]]]],
+    output_file: str,
+    debug: bool,
+):
+    """Dump the wrangled JSON to the output file.
+
+    :param dict[str, list[Optional[dict[str, Any]]]] mapped_data: The
+        mapped GCF data.
+    :param str output_file: The output filename.
+    :param bool debug: Whether debug mode is on.
+    """
+    if debug:
+        click.echo(f"📝 Output file {click.format_filename(output_file)}")
 
 
 if __name__ == "__main__":

gcf_data_mapper/cli.py

@@ -15,6 +15,6 @@ def collection(debug: bool) -> list[Optional[dict[str, Any]]]:
         Sheet.
     """
     if debug:
-        click.echo("📝 Wrangling GCF collection data.")
+        click.echo("📝 No GCF collection data to wrangle.")
 
     return []

gcf_data_mapper/parsers/collection.py

@@ -1,11 +1,13 @@
 from typing import Any, Optional
 
 import click
+import pandas as pd
 
 
-def document(debug: bool) -> list[Optional[dict[str, Any]]]:
+def document(mcf_docs: pd.DataFrame, debug: bool) -> list[Optional[dict[str, Any]]]:
     """Map the GCF document info to new structure.
 
+    :param pd.DataFrame mcf_docs: The MCF documents data.
     :param bool debug: Whether debug mode is on.
     :return list[Optional[dict[str, Any]]]: A list of GCF families in
         the 'destination' format described in the GCF Data Mapper Google

gcf_data_mapper/parsers/document.py

@@ -1,11 +1,14 @@
 from typing import Any, Optional
 
 import click
+import pandas as pd
 
 
-def family(debug: bool) -> list[Optional[dict[str, Any]]]:
+def family(projects_data: pd.DataFrame, debug: bool) -> list[Optional[dict[str, Any]]]:
     """Map the GCF family info to new structure.
 
+    :param pd.DataFrame projects_data: The MCF and GCF project data,
+        joined on FP num.
     :param bool debug: Whether debug mode is on.
     :return list[Optional[dict[str, Any]]]: A list of GCF families in
         the 'destination' format described in the GCF Data Mapper Google