Skip to content

Implement keyword argument support in METdbLoad.ush.met_db_load.main and allow calling the function from other Python modules #386

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
joelheaps wants to merge 6 commits into
base: develop
Choose a base branch
from
Draft

Implement keyword argument support in METdbLoad.ush.met_db_load.main and allow calling the function from other Python modules #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f079285
Implement keyword arguments for met_db_load.main and update tests
joelheaps Jun 21, 2025
0c67441
Add test outputs to .gitignore
joelheaps Jun 21, 2025
69b2671
Add missing dependencies to requirements.txt
joelheaps Jun 21, 2025
ec9705f
Fix "index only" test case
joelheaps Jun 21, 2025
0ab7291
Rename main function to load_met_data in met_db_load module
joelheaps Jun 23, 2025
21858e5
Rename main function to load_met_data in met_db_load module #1
joelheaps Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,7 @@ mpr_df_orig.txt

# other METplus components
/METcalcpy

# Test outputs
stdout/
tcmpr_reformatted.txt
169 changes: 75 additions & 94 deletions METdbLoad/test/test_met_db_load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pytest
import sys
from unittest.mock import patch, MagicMock
from METdbLoad.ush.met_db_load import main as load_main
from METdbLoad.ush.met_db_load import load_met_data
from METdbLoad.ush.met_db_load import purge_files, parse_args, next_set, print_version
from METdbLoad.ush.run_sql import RunSql
from METdbLoad.test.utils import dict_to_args
Expand Down Expand Up @@ -137,20 +137,17 @@ def test_met_db_table_counts(
"mtd_header_db_check": "true",
}

test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, load_flags=load_flags
)
),
"index": "true",
"tmpdir": [str(tmp_path)],
"loglevel": None,
}
)

load_main(test_args)
test_args = {
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, load_flags=load_flags
)
),
"index": "true",
"tmpdir": [str(tmp_path)],
}

load_met_data(**test_args)

for table, expected_count in expected_counts.items():
assert_count_rows(testRunSql.cur, table, expected_count)
Expand Down Expand Up @@ -206,22 +203,19 @@ def test_met_db_table_dups(
"mtd_header_db_check": "true",
"force_dup_file": "false",
}
test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, load_flags=load_flags
)
),
"index": "true",
"tmpdir": [str(tmp_path)],
"loglevel": None,
}
)

load_main(test_args)
test_args = {
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, load_flags=load_flags
)
),
"index": "true",
"tmpdir": [str(tmp_path)],
}

load_met_data(**test_args)
# load again to check duplicates aren't loaded in db
load_main(test_args)
load_met_data(**test_args)

for table, expected_count in expected_counts.items():
assert_count_rows(testRunSql.cur, table, expected_count)
Expand Down Expand Up @@ -277,22 +271,19 @@ def test_met_db_table_dups_allowed(
"mtd_header_db_check": "true",
"force_dup_file": "true",
}
test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, load_flags=load_flags
)
),
"index": "true",
"tmpdir": [str(tmp_path)],
"loglevel": None,
}
)

load_main(test_args)
test_args = {
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, load_flags=load_flags
)
),
"index": "true",
"tmpdir": [str(tmp_path)],
}

load_met_data(**test_args)
# load again to add duplicates
load_main(test_args)
load_met_data(**test_args)

for table, expected_count in expected_counts.items():
assert_count_rows(testRunSql.cur, table, expected_count * 2)
Expand All @@ -304,29 +295,26 @@ def test_met_db_indexes(
tmp_path,
):
# set up to "apply_indexes"
test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(
tmp_path,
POINT_STAT_DATA_DIR,
"point_stat",
{"apply_indexes": "true"},
)
),
"index": "false",
"tmpdir": [str(tmp_path)],
"loglevel": None,
}
)
test_args = {
"xmlfile": str(
get_xml_test_file(
tmp_path,
POINT_STAT_DATA_DIR,
"point_stat",
{"apply_indexes": "true"},
)
),
"index": True,
"tmpdir": [str(tmp_path)],
}

# file_id and stat_header are already indexed
idx_cnt = testRunSql.cur.execute("SHOW INDEX FROM line_data_fho")
assert idx_cnt == 2

# sys.exit is called after processing indexes
with pytest.raises(SystemExit):
load_main(test_args)
load_met_data(**test_args)

# check extra indicies have been created
idx_cnt = testRunSql.cur.execute("SHOW INDEX FROM line_data_fho")
Expand All @@ -338,7 +326,7 @@ def test_met_db_indexes(
# check sys.exit called on error
with pytest.raises(SystemExit):
with patch.object(RunSql, "apply_indexes", side_effect=KeyError):
load_main(test_args)
load_met_data(**test_args)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -395,27 +383,24 @@ def test_local_in_file(
):
"""check we get the same result when local_file is on or off"""

test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, local_infile=local_infile
)
),
"index": "false",
"tmpdir": [str(tmp_path)],
"loglevel": None,
}
)

load_main(test_args)
test_args = {
"xmlfile": str(
get_xml_test_file(
tmp_path, met_data_dir, met_tool, local_infile=local_infile
)
),
"index": False,
"tmpdir": [str(tmp_path)],
}

load_met_data(**test_args)

for table, expected_count in expected_counts.items():
assert_count_rows(testRunSql.cur, table, expected_count)


def test_empty_files(tmp_path):
"""Junk files shouldn't cause an error when running load_main"""
"""Junk files shouldn't cause an error when running load_met_data"""

met_data_dir = tmp_path / "empty_files_test"
met_data_dir.mkdir()
Expand All @@ -434,23 +419,19 @@ def test_empty_files(tmp_path):
"VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE\n"
)

test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(
tmp_path,
met_data_dir,
"mtd",
)
),
"index": "false",
"tmpdir": [str(tmp_path)],
"loglevel": None,
}
)

load_main(test_args)
test_args = {
"xmlfile": str(
get_xml_test_file(
tmp_path,
met_data_dir,
"mtd",
)
),
"index": False,
"tmpdir": [str(tmp_path)]
}

load_met_data(**test_args)

def test_print_version():
mock_logger = MagicMock()
Expand All @@ -459,7 +440,7 @@ def test_print_version():
assert mock_logger.info.call_args[0][0].startswith("METdbload Version:")

with pytest.raises(SystemExit):
with patch("os.path.dirname", side_effect=TypeError):
with patch("METdbLoad.ush.met_db_load.PACKAGE_NAME", "metdataio_bogus_package_name"):
print_version(mock_logger)
assert mock_logger.error.call_count == 2

Expand Down
22 changes: 10 additions & 12 deletions METdbLoad/test/test_run_sql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from unittest import mock
from pymysql import OperationalError

from METdbLoad.ush.met_db_load import main as load_main
from METdbLoad.ush.met_db_load import load_met_data
from METdbLoad.ush import constants as CN
from METdbLoad.test.utils import dict_to_args
from METdataio.METdbLoad.test.utils import (
Expand All @@ -14,17 +14,15 @@
def populate_some_data(
tmp_path, met_data_dir=GRID_STAT_DATA_DIR, met_tool="grid_stat", load_flags={}
):
test_args = dict_to_args(
{
"xmlfile": str(
get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags)
),
"index": "true",
"tmpdir": [str(tmp_path)],
"loglevel": "DEBUG"
}
)
load_main(test_args)
test_args = {
"xmlfile": str(
get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags)
),
"index": "true",
"tmpdir": [str(tmp_path)],
"loglevel": "DEBUG"
}
load_met_data(**test_args)


def test_get_file_name(tmp_path, emptyDB, testRunSql, mock_logger):
Expand Down
Loading