diff --git a/README.md b/README.md index 10c87d48..9d45bae8 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,26 @@ Python tools for MMD. The package contains tools for generating MMD files from n generates an output MMD file called `reference_nc.xml`. +In addition, the `mmd_operations` module currently contains a tool to move data +files and accordingly update MMD files registered in online catalogs. This +module can be extended with other necessary data management tools. Moving +can be done with the `move_data` script, e.g.: + +``` +move_data /path/to/files-from-git/mmd-xml- /path/to/old/storage /path/to/new/storage "%Y/%m/%d/*.nc" --dmci-update +``` + +The two last arguments provide a search pattern in case the netCDF files are +stored in subfolders, and to directly updated the metadata catalog, +respectively. If --dmci-update is not provided, local MMD files will not be +pushed to the catalog. + +The results of the `move_data` script will be logged to a file, which by +default is named `move_data.log`. You can change the filename through the +option `--log-file`. Due to a bug in pycsw, the file may contain warnings about +not found datasets. This can be handled by reingesting the MMD files. Keep the +log file, and get help from a data manager to handle this. + # Installation To avoid problems with conflicting versions, we recommend using the [Conda]( diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py new file mode 100644 index 00000000..f90a4811 --- /dev/null +++ b/py_mmd_tools/mmd_operations.py @@ -0,0 +1,252 @@ +""" +License: + +This file is part of the py-mmd-tools repository +. + +py-mmd-tools is licensed under the Apache License 2.0 + +""" +import os +import pytz +import uuid +import shutil +import logging +import netCDF4 +import datetime +import requests +import tempfile +import datetime_glob + +import urllib.parse + + +def add_metadata_update_info(f, note, type="Minor modification"): + """ Add update information """ + f.write( + " \n" + " %s\n" + " %s\n" + " %s\n" + " \n" % (datetime.datetime.utcnow().replace( + tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), type, note)) + + +def check_csw_catalog(ds_id, nc_file, urls, env, emsg=""): + """Search for the dataset with id 'ds_id' in the CSW metadata + catalog. + """ + payload = { + "service": "CSW", + "version": "2.0.2", + "request": "GetRepositoryItem", + "id": ds_id} + + payload_str = urllib.parse.urlencode(payload, safe=":") + + ds_found_and_accessible = False + res = requests.get(url=f"https://{urls[env]['csw']}/csw", + params=payload_str) + # TODO: check the data_access urls + if res.status_code == 200 and "ExceptionText" not in res.text: + ds_found_and_accessible = True + else: + emsg += f"Could not find dataset ({ds_id}) in CSW catalog: {nc_file}, {res.text}" + + return ds_found_and_accessible, emsg + + +def get_local_mmd_git_path(nc_file, mmd_repository_path): + """Return the path to the original MMD file. + """ + ds = netCDF4.Dataset(nc_file) + lvlA = "arch_%s" % uuid.UUID(ds.id).hex[7] + lvlB = "arch_%s" % uuid.UUID(ds.id).hex[6] + lvlC = "arch_%s" % uuid.UUID(ds.id).hex[5] + mmd_filename = ds.id + ".xml" + ds.close() + return os.path.join(mmd_repository_path, lvlA, lvlB, lvlC, mmd_filename) + + +def mmd_change_file_location(mmd, new_file_location, copy=True): + """Copy original MMD file, and change the file_location field. + Return the filename of the updated MMD file, and a status flag + indicating if it has been changed or not. + """ + if not os.path.isfile(mmd): + raise ValueError(f"File does not exist: {mmd}") + if copy: + tmp_path = tempfile.gettempdir() + shutil.copy2(mmd, tmp_path) + # Edit copied MMD file + mmd = os.path.join(tmp_path, os.path.basename(mmd)) + lines = mmd_readlines(mmd) + # Open the MMD file and add updates + status = False + with open(mmd, "w") as f: + for line in lines: + if "" in line: + add_metadata_update_info(f, "Change storage information.") + if "" in line: + f.write(f" {new_file_location}\n") + status = True + else: + f.write(line) + return mmd, status + + +def mmd_readlines(filename): + """ Read lines in MMD file. + """ + if not os.path.exists(filename): + raise ValueError("File does not exist: %s" % filename) + with open(filename, "r") as f: + lines = f.readlines() + return lines + + +def move_data_file(nc_file, nfl, emsg=""): + """Move data file from nc_file to nfl. + """ + nc_moved = False + try: + shutil.move(nc_file, nfl) + except Exception as e: + nc_moved = False + emsg = f"Could not move file from {nc_file} to {nfl}.\nError message: {str(e)}\n" + else: + nc_moved = True + return nc_moved, emsg + + +def move_data(mmd_repository_path, old_file_location_base, new_file_location_base, + ext_pattern=None, dry_run=True, env="prod"): + """Update MMD and move data file. + """ + if env not in ["dev", "staging", "prod"]: + raise ValueError("Invalid env input") + if env not in mmd_repository_path: + raise ValueError("Invalid mmd_repository path") + + urls = { + "prod": { + "dmci": "dmci.s-enda.k8s.met.no", + "csw": "data.csw.met.no", + "id_namespace": "no.met", + }, + "staging": { + "dmci": "dmci.s-enda-staging.k8s.met.no", + "csw": "https://csw.s-enda-staging.k8s.met.no/", + "id_namespace": "no.met.staging", + }, + "dev": { + "dmci": "dmci.s-enda-dev.k8s.met.no", + "csw": "https://csw.s-enda-dev.k8s.met.no/", + "id_namespace": "no.met.dev", + } + } + + if os.path.isfile(old_file_location_base): + existing = [old_file_location_base] + else: + existing = [str(nc_file) for match, nc_file in + datetime_glob.walk(pattern=os.path.join(old_file_location_base, ext_pattern))] + + copy_mmd = True + if dry_run: + # Not copying the MMD file will make it easy to check changes + # with git diff + copy_mmd = False + + updated = [] + not_updated = {} + for nc_file in existing: + # Error message + emsg = "" + nfl = new_file_location(nc_file, new_file_location_base, old_file_location_base, dry_run) + mmd_orig = get_local_mmd_git_path(nc_file, mmd_repository_path) + + # Check permissions before doing anything + remove_file_allowed = os.access(nc_file, os.W_OK) + write_file_allowed = os.access(nfl, os.W_OK) + if not remove_file_allowed or not write_file_allowed: + if not remove_file_allowed and not write_file_allowed: + raise PermissionError(f"Missing permissions to delete {nc_file} " + f"and to write {nfl}") + if not remove_file_allowed: + raise PermissionError(f"Missing permission to delete {nc_file}") + if not write_file_allowed: + raise PermissionError(f"Missing permission to write {nfl}") + + mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl, copy=copy_mmd) + if not mmd_updated: + raise Exception(f"Could not update MMD file for {nc_file}") + + # Get MMD content as binary data + with open(mmd_new, "rb") as fn: + data = fn.read() + res = requests.post(url=f"https://{urls[env]['dmci']}/v1/validate", data=data) + + # Update with dmci update + dmci_updated = False + if res.status_code == 200 and "OK" in res.text and not dry_run: + ds = netCDF4.Dataset(nc_file) + ds_id = f"{ds.naming_authority}:{ds.id}".strip() + # Delete dataset + del_res = requests.post(url=f"https://{urls[env]['dmci']}/v1/delete/{ds_id}") + if del_res.status_code != 200 or "OK" not in del_res.text: + raise Exception(f"Not able to delete dataset ({ds_id}): {del_res.text}") + # Reingest dataset + res = requests.post(url=f"https://{urls[env]['dmci']}/v1/insert", data=data) + """NOTE: because of a bug in pycsw, the below updated is + replaced by delete and insert above + """ + # Update dataset + # res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data) + if res.status_code == 200 and "OK" in res.text: + # This should be the case for a dry-run and a valid xml + dmci_updated = True + else: + raise Exception("Could not push updated MMD file to the " + f"DMCI API: {mmd_new}, {res.text}") + + if dmci_updated and not dry_run: + nc_moved, emsg = move_data_file(nc_file, nfl) + if not nc_moved: + raise Exception(f"Could not move {nc_file} to {nfl}.") + elif dmci_updated and dry_run: + nc_moved = True + + ds_id = f"{urls[env]['id_namespace']}:{os.path.basename(mmd_orig).split('.')[0]}" + if not dry_run: + ds_found_and_accessible, emsg = check_csw_catalog(ds_id, nc_file, urls, env, emsg=emsg) + else: + ds_found_and_accessible = True + + if not ds_found_and_accessible: + logging.warning(emsg) + + if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]): + updated.append(mmd_orig) + logging.info(f"Updated {mmd_orig}.") + else: + not_updated[mmd_orig] = emsg + + return not_updated, updated + + +def new_file_location(nc_file, new_base_loc, existing_base_loc, dry_run): + """Return the name of the new folder where the netcdf file will be + stored. Subfolders of new_base_loc will be created. + """ + if not os.path.isdir(new_base_loc): + raise ValueError(f"Folder does not exist: {new_base_loc}") + file_path = nc_file.replace(existing_base_loc, new_base_loc) + new_folder = os.path.dirname(os.path.abspath(file_path)) + if not dry_run: + try: + os.makedirs(new_folder) + except FileExistsError: + # Do nothing + pass + return new_folder diff --git a/py_mmd_tools/mmd_to_nc.py b/py_mmd_tools/mmd_to_nc.py index 0d52817d..fb74bac8 100644 --- a/py_mmd_tools/mmd_to_nc.py +++ b/py_mmd_tools/mmd_to_nc.py @@ -119,7 +119,7 @@ def process_element(self, xml_element, translations): raise ValueError('Multiple ACDD or ACCD extension fields provided.' ' Please use another translation function.') # Update the dictionary containing the ACDD elements - self.update_acdd({acdd_name[0]: xml_element.text}, {acdd_name[0]: sep}) + self.update_acdd({acdd_name[0]: xml_element.text}, {acdd_name[0]: sep[0]}) def update_acdd(self, new_dict, sep=None): """ diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py new file mode 100755 index 00000000..528ffbea --- /dev/null +++ b/py_mmd_tools/script/move_data.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Script to move one or more datasets from one location to another, and +update its MMD xml file accordingly. + +License: + +This file is part of the py-mmd-tools repository +. + +py-mmd-tools is licensed under the Apache License 2.0 + +""" +import os +import logging +import argparse + +from py_mmd_tools.mmd_operations import move_data + + +def create_parser(): + """Create parser object""" + parser = argparse.ArgumentParser(description="Move one or more datasets from one location to " + "another, and update its MMD xml file " + "accordingly.") + parser.add_argument( + "mmd_repository_path", type=str, + help="Local folder containing all MMD files.") + parser.add_argument( + "old_file_location_base", type=str, + help="Base folder from which the data file(s) will be moved, or exact path to a file.") + parser.add_argument( + "new_file_location_base", type=str, + help="Base or exact path to the folder to which the data file(s) will be moved.") + parser.add_argument( + "--ext-pattern", type=str, default=None, + help="Pathname pattern extending old_file_location_base, i.e., extending the " + "existing file *base* location(s) with, e.g, the year and month as a " + "glob pattern intertwined with date/time format akin to " + "strptime/strftime format (e.g., '%Y/%m').") + parser.add_argument( + "--dmci-update", action="store_true", + help="Directly update the online catalog with the changed MMD files." + ) + parser.add_argument( + "--log-file", type=str, default="move_data.log", + help="Log filename") + + return parser + + +def main(args=None): + """Move dataset(s) and update MMD. + """ + if not os.path.isdir(args.mmd_repository_path): + raise ValueError(f"Invalid input: {args.mmd_repository_path}") + + if not os.path.isdir(args.new_file_location_base): + raise ValueError(f"Invalid input: {args.new_file_location_base}") + + logging.basicConfig(filename=args.log_file, level=logging.INFO) + + not_updated, updated = move_data(args.mmd_repository_path, + args.old_file_location_base, + args.new_file_location_base, + args.ext_pattern, + dry_run=not args.dmci_update) + + return updated, not_updated + + +def _main(): # pragma: no cover + main(create_parser().parse_args()) # entry point in setup.cfg + + +if __name__ == '__main__': # pragma: no cover + main(create_parser().parse_args()) diff --git a/pyproject.toml b/pyproject.toml index 8ab8b446..c0115d6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "shapely", "wget", "xmltodict", + "datetime_glob", ] name = "py-mmd-tools" description = "This is a tools for generating MMD files from netCDF-CF files with ACDD attributes, for documenting netCDF-CF files from MMD information." @@ -49,6 +50,7 @@ nc2mmd = "py_mmd_tools.script.nc2mmd:_main" check_nc = "py_mmd_tools.script.check_nc:_main" yaml2adoc = "py_mmd_tools.script.yaml2adoc:_main" ncheader2json = "py_mmd_tools.script.ncheader2json:_main" +move_data = "py_mmd_tools.script.move_data:_main" [project.urls] source = "https://github.com/metno/py-mmd-tools" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..d90899d6 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + py_mmd_tools: Core functionality tests + online: Tests requiring web access diff --git a/tests/data/2024/09/01/reference_nc.nc b/tests/data/2024/09/01/reference_nc.nc new file mode 100644 index 00000000..9e053c5c Binary files /dev/null and b/tests/data/2024/09/01/reference_nc.nc differ diff --git a/tests/data/reference_nc.xml b/tests/data/reference_nc.xml index cdeb0bf6..27c768f9 100644 --- a/tests/data/reference_nc.xml +++ b/tests/data/reference_nc.xml @@ -6,17 +6,26 @@ Norsk abstrakt. Active In Work - ADC METNCS 2020-11-27T14:05:56Z - Created - 2020-11-27T14:05:56Z - Created + 2024-07-30T12:36:53Z + Minor modification + New file location in storage information. + + + 2024-07-30T12:39:39Z + Minor modification + New file location in storage information. + + + 2024-07-30T12:39:57Z + Minor modification + New file location in storage information. @@ -43,6 +52,11 @@ https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling + + toa_bidirectional_reflectance + https://vocab.met.no/mmd/Keywords_Vocabulary/CFSTDN + + 77.96752166748047 @@ -116,7 +130,11 @@ post@met.no Norwegian Meteorological Institute - NORWAY + Meteorologisk institutt, Henrik Mohnsplass 1 + Oslo + Oslo + 0000 + Norway @@ -126,13 +144,34 @@ met.no + No quality control + + OPeNDAP + Open-source Project for a Network Data Access Protocol + https://thredds.met.no/thredds/dodsC/reference_nc.nc + + + HTTP + Direct download of file + https://thredds.met.no/thredds/fileServer/reference_nc.nc + reference_nc.nc - tests/data/reference_nc.nc + /some/where/new NetCDF-CF - 0.02 - + 0.03 + aa188b45ece9bdbbc9b470106a6b13f8 + + Dataset landing page + + https://data.met.no/dataset/b7cb7934-77ca-4439-812e-f560df3fe7eb + + + Scientific publication + + https://ieeexplore.ieee.org/document/7914752 + METNCS MET Norway core services @@ -148,11 +187,12 @@ grid + Not available DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING 2020-11-27T14:05:56Z Direct Broadcast data processed in satellite swath to L1C. - https://met.no/the/dataset/landing-page - https://met.no/published/or/web-based/references/that/describe/the/data/or/methods/used/to/produce/it/should/be/URI + Norwegian Meteorological Institute + https://data.met.no/dataset/b7cb7934-77ca-4439-812e-f560df3fe7eb diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py new file mode 100644 index 00000000..1040b9e9 --- /dev/null +++ b/tests/test_mmd_operations.py @@ -0,0 +1,355 @@ +""" +License: + +This file is part of the py-mmd-tools repository +. + +py-mmd-tools is licensed under the Apache License 2.0 + +""" +import os +import pytest +import shutil +import subprocess + +from unittest.mock import Mock + +from py_mmd_tools.mmd_operations import check_csw_catalog +from py_mmd_tools.mmd_operations import move_data +from py_mmd_tools.mmd_operations import move_data_file +from py_mmd_tools.mmd_operations import mmd_readlines +from py_mmd_tools.mmd_operations import new_file_location +from py_mmd_tools.mmd_operations import mmd_change_file_location +from py_mmd_tools.mmd_operations import get_local_mmd_git_path + + +@pytest.mark.py_mmd_tools +def test_get_local_mmd_git_path(dataDir): + """Test that the mmd git path is returned correctly with folders + composed from its uuid. + """ + ncfile = os.path.join(dataDir, "reference_nc.nc") + fn = get_local_mmd_git_path(ncfile, "/some/folder/mmd-xml-production") + assert fn == "/some/folder/mmd-xml-production/arch_4/arch_3/arch_9/" \ + "b7cb7934-77ca-4439-812e-f560df3fe7eb.xml" + + +@pytest.mark.py_mmd_tools +def test_mmd_change_file_location(dataDir, monkeypatch): + """Test that an MMD file is created with new file_location, and + new metadata update. + """ + mmd = os.path.join(dataDir, "reference_nc_TMP.xml") + shutil.copy(os.path.join(dataDir, "reference_nc.xml"), mmd) + new_file_location = "/some/where/else/2024/06/19" + + new_mmd, changed = mmd_change_file_location(mmd, new_file_location) + assert changed is True + assert os.path.isfile(new_mmd) + lines = mmd_readlines(new_mmd) + for line in lines: + if "" in line: + assert "/some/where/else/2024/06/19" in line + + mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False) + assert changed is True + assert os.path.isfile(mmd) + lines = mmd_readlines(mmd) + for line in lines: + if "file_location" in line: + assert "/some/where/else/2024/06/19" in line + + # Delete tmp files + os.remove(mmd) + os.remove(new_mmd) + + # Test that it fails when the mmd does not exist + with pytest.raises(ValueError): + mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False) + assert mmd is None + assert changed is False + + +@pytest.mark.py_mmd_tools +def test_mmd_readlines(dataDir): + """Test that the lines in the MMD file are returned as a list. + """ + mmd = os.path.join(dataDir, "reference_nc.xml") + lines = mmd_readlines(mmd) + assert isinstance(lines, list) + assert "mmd:mmd" in lines[0] + + mmd = os.path.join(dataDir, "does_not_exist.xml") + with pytest.raises(ValueError): + lines = mmd_readlines(mmd) + + +@pytest.mark.py_mmd_tools +def test_move_data(dataDir, monkeypatch, caplog): + """Test the move_data function. + """ + mmd_repository_path = "/some/folder/mmd-xml-production" + old_file_location_base = dataDir + new_file_location_base = "/some/where/new" + nc_file = os.path.join(dataDir, "reference_nc.nc") + + def mock_walk(*a, **k): + yield (1, nc_file) + + # This variable is needed when mocking datetime_glob.walk + pattern = os.path.join(dataDir, "%Y/%m/%d/*.nc") + + class MockResponse: + status_code = 200 + text = "OK" + + # Test check for environment in move_data function + with pytest.raises(ValueError): + move_data(mmd_repository_path, new_file_location_base, nc_file, env="hei") + + # Check that an error is raised if the given env is not in the MMD repo path + with pytest.raises(ValueError): + move_data("/some/folder/mmd-xml-noenv", new_file_location_base, nc_file, env="dev") + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.new_file_location", + lambda *a, **k: new_file_location_base) + mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path", + lambda *a, **k: os.path.join(dataDir, "reference_nc.xml")) + mp.setattr("py_mmd_tools.mmd_operations.shutil.move", + lambda *a, **k: None) + mp.setattr("py_mmd_tools.mmd_operations.requests.get", + lambda *a, **k: MockResponse()) + mp.setattr("py_mmd_tools.mmd_operations.requests.post", + lambda *a, **k: MockResponse()) + mp.setattr("py_mmd_tools.mmd_operations.os.access", + lambda *a, **k: True) + # Successful call + not_updated, updated = move_data(mmd_repository_path, nc_file, new_file_location_base) + assert len(not_updated) == 0 + assert len(updated) == 1 + assert os.path.isfile(updated[0]) + lines = mmd_readlines(updated[0]) + for line in lines: + if "" in line: + assert "/some/where/new" in line + + # The ext_pattern input is tested in test_move_data_script + + # Test that an exception is raised if we're not able to move the nc-file + mp.setattr("py_mmd_tools.mmd_operations.move_data_file", lambda *a, **k: (False, "")) + with pytest.raises(Exception): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern, dry_run=False) + + # Test that an exception is raised if the MMD file cannot be + # updated with a new netCDF file location + mp.setattr("py_mmd_tools.mmd_operations.mmd_change_file_location", + lambda *a, **k: (nc_file, False)) + with pytest.raises(Exception): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern) + + def raise_(ex): + raise ex + + # Test that an error is raised if we cannot find the MMD path + mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk", + lambda *a, **k: mock_walk(*a, **k)) + mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path", + lambda *a, **k: raise_(Exception("No path"))) + with pytest.raises(Exception): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern) + + # Test os.access, remove_file_allowed is False + mock_access = Mock() + mock_access.side_effect = [False, True] + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk", + lambda *a, **k: mock_walk(*a, **k)) + mp.setattr("py_mmd_tools.mmd_operations.new_file_location", + lambda *a, **k: new_file_location_base) + mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access) + with pytest.raises(PermissionError): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern) + + # Test os.access, write_file_allowed is False + mock_access = Mock() + mock_access.side_effect = [True, False] + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk", + lambda *a, **k: mock_walk(*a, **k)) + mp.setattr("py_mmd_tools.mmd_operations.new_file_location", + lambda *a, **k: new_file_location_base) + mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access) + with pytest.raises(PermissionError): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern) + + # Test os.access, remove_file_allowed and write_file_allowed are + # False + mock_access = Mock() + mock_access.side_effect = [False, False] + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk", + lambda *a, **k: mock_walk(*a, **k)) + mp.setattr("py_mmd_tools.mmd_operations.new_file_location", + lambda *a, **k: new_file_location_base) + mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access) + with pytest.raises(PermissionError): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern) + + # Test second call to requests.post fails (dmci update) + class MockResponseFail: + + status_code = 400 + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk", + lambda *a, **k: mock_walk(*a, **k)) + mp.setattr("py_mmd_tools.mmd_operations.new_file_location", + lambda *a, **k: new_file_location_base) + mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path", + lambda *a, **k: os.path.join(dataDir, "reference_nc.xml")) + mp.setattr("py_mmd_tools.mmd_operations.shutil.move", + lambda *a, **k: None) + mp.setattr("py_mmd_tools.mmd_operations.os.access", + lambda *a, **k: True) + mp.setattr("py_mmd_tools.mmd_operations.requests.get", + lambda *a, **k: MockResponse()) + + # Check that an exception is raised if delete fails + mock_post = Mock() + mock_post.side_effect = [MockResponse(), MockResponseFail()] + mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post) + with pytest.raises(Exception): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern, dry_run=False) + + # Check that an exception is raised if insert fails + mock_post = Mock() + mock_post.side_effect = [MockResponse(), MockResponse(), MockResponseFail()] + mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post) + with pytest.raises(Exception): + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern, dry_run=False) + + # Check that updated is False if check_csw_catalog fails + mock_post = Mock() + mock_post.side_effect = [MockResponse(), MockResponse(), MockResponse()] + mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post) + mp.setattr("py_mmd_tools.mmd_operations.check_csw_catalog", + lambda *a, **k: (False, "Fail")) + not_updated, updated = move_data(mmd_repository_path, old_file_location_base, + new_file_location_base, pattern, dry_run=False) + assert "Fail" in caplog.record_tuples[0][2] + assert not_updated[os.path.join(dataDir, "reference_nc.xml")] == "Fail" + + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + +@pytest.mark.py_mmd_tools +def test_new_file_location(monkeypatch): + """Test that the returned new location paths are correct. + """ + file = "/some/old/loc/2024/06/19/" \ + "S1A_IW_GRDM_1SDV_20240619T053156_20240619T053223_054388_069E03_FC95_MEPS.nc" + new_base = "/some/where/else" + existing_base_loc = "/some/old/loc" + + with pytest.raises(ValueError): + new_file_location(file, new_base, existing_base_loc, True) + + def raise_(ex): + raise ex + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", lambda *a, **k: True) + mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", lambda *a, **k: None) + assert new_file_location(file, new_base, existing_base_loc, False) == \ + "/some/where/else/2024/06/19" + + mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", + lambda *a, **k: raise_(FileExistsError)) + assert new_file_location(file, new_base, existing_base_loc, False) == \ + "/some/where/else/2024/06/19" + + +@pytest.mark.py_mmd_tools +def test_check_csw_catalog(monkeypatch): + """Test check_csw_catalog + """ + ds_id = "no.met:123" + nc_file = "/some/file.nc" + urls = { + "prod": { + "csw": "data.csw.some-place.no", + } + } + env = "prod" + + class MockResponse: + + status_code = 400 + text = "Fail" + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.requests.get", + lambda *a, **k: MockResponse()) + found, msg = check_csw_catalog(ds_id, nc_file, urls, env) + assert found is False + assert msg == "Could not find dataset (no.met:123) in CSW catalog: /some/file.nc, Fail" + + class MockResponse2: + + status_code = 200 + text = "" + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.requests.get", + lambda *a, **k: MockResponse2()) + found, msg = check_csw_catalog(ds_id, nc_file, urls, env) + assert found is True + assert msg == "" + + +@pytest.mark.online +def test_check_dataset_in_met_csw_catalog(caplog): + """Check that a known dataset is found. + """ + ds_id = "no.met:806070da-e9f3-4d03-ba1d-26b843961634" + nc_file = "ncfile.nc" + urls = {"prod": {"dmci": "dmci.s-enda.k8s.met.no", + "csw": "data.csw.met.no", + "id_namespace": "no.met"}} + env = "prod" + found, msg = check_csw_catalog(ds_id, nc_file, urls, env) + assert found is True + assert msg == "" + + ds_id = "rubbish" + found, msg = check_csw_catalog(ds_id, nc_file, urls, env) + assert found is False + + +@pytest.mark.py_mmd_tools +def test_move_data_file(monkeypatch): + """Test move_data_file function can't move (working move is + tested in test_move_data). + """ + nc_file = "/some/file.nc" + nfl = "/some/new/location" + + def raise_(ex): + raise ex + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.shutil.move", + lambda *a, **k: raise_(Exception("Permission error..."))) + moved, msg = move_data_file(nc_file, nfl) + assert moved is False diff --git a/tests/test_mmd_to_nc.py b/tests/test_mmd_to_nc.py index 6553131f..6900710b 100644 --- a/tests/test_mmd_to_nc.py +++ b/tests/test_mmd_to_nc.py @@ -417,7 +417,7 @@ def test_process_element_2(self): md.process_element(element_to_translate, md.mmd_yaml) self.assertIsNone(md.acdd_metadata) # MMD element listed in the translation dictionary, but with no translation information - element_to_translate = md.tree.find('mmd:last_metadata_update/mmd:update/mmd:note', + element_to_translate = md.tree.find('mmd:last_metadata_update/mmd:update/mmd:type', md.namespaces) md.process_element(element_to_translate, md.mmd_yaml) self.assertIsNone(md.acdd_metadata) diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py new file mode 100644 index 00000000..1b452924 --- /dev/null +++ b/tests/test_move_data_script.py @@ -0,0 +1,164 @@ +""" +License: + +This file is part of the py-mmd-tools repository +. + +py-mmd-tools is licensed under the Apache License 2.0 + +""" +import os +import pytest +import subprocess + +from py_mmd_tools.script.move_data import main +from py_mmd_tools.script.move_data import create_parser + + +@pytest.mark.py_mmd_tools +def test_main(dataDir, monkeypatch, caplog): + """ + """ + mmd_repository_path = "/some/folder/mmd-xml-production" + old_file_location_base = os.path.join(dataDir, "reference_nc.nc") + new_file_location_base = "/some/where/new" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + ]) + + class MockResponse: + + status_code = 200 + text = "OK" + + with monkeypatch.context() as mp: + mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", + lambda *a, **k: True) + mp.setattr("py_mmd_tools.mmd_operations.new_file_location", + lambda *a, **k: new_file_location_base) + mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path", + lambda *a, **k: os.path.join(dataDir, "reference_nc.xml")) + mp.setattr("py_mmd_tools.mmd_operations.requests.get", + lambda *a, **k: MockResponse()) + mp.setattr("py_mmd_tools.mmd_operations.requests.post", + lambda *a, **k: MockResponse()) + mp.setattr("py_mmd_tools.mmd_operations.shutil.move", lambda *a, **k: None) + mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", lambda *a, **k: None) + mp.setattr("py_mmd_tools.mmd_operations.os.access", + lambda *a, **k: True) + u, n = main(parsed) + assert len(u) == 1 + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", + lambda *a, **k: False) + with pytest.raises(ValueError): + main(parsed) + + map = { + mmd_repository_path: True, + new_file_location_base: False, + } + + def mock_isdir(pp): + return map[pp] + + mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", mock_isdir) + with pytest.raises(ValueError): + main(parsed) + + map = { + mmd_repository_path: True, + new_file_location_base: True, + } + mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", mock_isdir) + + old_file_location_base = dataDir + + """ The following tests are basically testing datetime_glob.walk + but are included anyway... + """ + # Day as format code + ext_pattern = "2024/09/%d/*.nc" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + "--ext-pattern", ext_pattern, + ]) + u, n = main(parsed) + assert len(u) == 1 + + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + # Month and day as format codes + ext_pattern = "2024/%m/%d/*.nc" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + "--ext-pattern", ext_pattern, + ]) + u, n = main(parsed) + assert len(u) == 1 + + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + # Year, month and day as format codes + ext_pattern = "%Y/%m/%d/*.nc" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + "--ext-pattern", ext_pattern, + ]) + u, n = main(parsed) + assert len(u) == 1 + + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + # Month as format code + ext_pattern = "2024/%m/01/*.nc" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + "--ext-pattern", ext_pattern, + ]) + u, n = main(parsed) + assert len(u) == 1 + + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + # Year as format code + ext_pattern = "%Y/09/01/*.nc" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + "--ext-pattern", ext_pattern, + ]) + u, n = main(parsed) + assert len(u) == 1 + + subprocess.run(["git", "restore", "tests/data/reference_nc.xml"]) + + # No match + ext_pattern = "%Y/09/02/*.nc" + parser = create_parser() + parsed = parser.parse_args([ + mmd_repository_path, + old_file_location_base, + new_file_location_base, + "--ext-pattern", ext_pattern, + ]) + u, n = main(parsed) + assert len(u) == 0 diff --git a/tests/test_nc_to_mmd.py b/tests/test_nc_to_mmd.py index ff3e31ae..fed2d75c 100644 --- a/tests/test_nc_to_mmd.py +++ b/tests/test_nc_to_mmd.py @@ -153,14 +153,15 @@ def test_separate_repeated(dataDir): """ Test function Nc_to_mmd.separate_repeated """ md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = ['"Basis": "Space-based Platforms"', - ' "Category": "Earth Observation Satellites"', - ' "Sub_Category": "Sentinel-1"', - ' "Short_Name": "Sentinel-1A"', - ' "Long_Name": "Sentinel-1A"'] - with pytest.raises(AttributeError) as ee: - md.separate_repeated(True, getattr(ncin, "platform")) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = [""Basis": "Space-based Platforms"", + " "Category": "Earth Observation Satellites"", + " "Sub_Category": "Sentinel-1"", + " "Short_Name": "Sentinel-1A"", + " "Long_Name": "Sentinel-1A""] + with pytest.raises(AttributeError) as ee: + md.separate_repeated(True, getattr(ncin, "platform")) + assert str(ee.value) == "'list' object has no attribute 'split'" @@ -170,20 +171,21 @@ def testNc_to_mmd_get_geographic_extent_polygon(dataDir): geospatial_bounds_crs is missing. """ md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.geospatial_bounds = ("POLYGON ((59.01 1.23, 59.06 1.66, 59.10 2.09, 59.15 2.53, " - "59.19 2.96, 59.24 3.40, 59.28 3.84, 59.32 4.28, 59.35 4.72, " - "59.39 5.16, 59.43 5.61, 59.43 5.61, 59.57 5.57, 59.72 5.52, " - "59.87 5.48, 60.01 5.43, 60.16 5.39, 60.31 5.34, 60.46 5.29, " - "60.60 5.25, 60.75 5.20, 60.92 5.15, 60.92 5.15, 60.88 4.67, " - "60.84 4.21, 60.80 3.75, 60.76 3.29, 60.72 2.84, 60.68 2.38, " - "60.63 1.92, 60.59 1.47, 60.54 1.02, 60.49 0.56, 60.49 0.56, " - "60.33 0.64, 60.18 0.71, 60.03 0.77, 59.89 0.84, 59.74 0.90, " - "59.59 0.97, 59.45 1.03, 59.30 1.10, 59.15 1.16, 59.01 1.23))") - mmd_yaml = yaml.load( - resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader - ) - data = md.get_geographic_extent_polygon(mmd_yaml["geographic_extent"].pop("polygon"), ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.geospatial_bounds = ("POLYGON ((59.01 1.23, 59.06 1.66, 59.10 2.09, 59.15 2.53, " + "59.19 2.96, 59.24 3.40, 59.28 3.84, 59.32 4.28, 59.35 4.72, " + "59.39 5.16, 59.43 5.61, 59.43 5.61, 59.57 5.57, 59.72 5.52, " + "59.87 5.48, 60.01 5.43, 60.16 5.39, 60.31 5.34, 60.46 5.29, " + "60.60 5.25, 60.75 5.20, 60.92 5.15, 60.92 5.15, 60.88 4.67, " + "60.84 4.21, 60.80 3.75, 60.76 3.29, 60.72 2.84, 60.68 2.38, " + "60.63 1.92, 60.59 1.47, 60.54 1.02, 60.49 0.56, 60.49 0.56, " + "60.33 0.64, 60.18 0.71, 60.03 0.77, 59.89 0.84, 59.74 0.90, " + "59.59 0.97, 59.45 1.03, 59.30 1.10, 59.15 1.16, 59.01 1.23))") + mmd_yaml = yaml.load( + resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader + ) + data = md.get_geographic_extent_polygon(mmd_yaml["geographic_extent"].pop("polygon"), ncin) + assert data["srsName"] == "EPSG:4326" @@ -196,49 +198,44 @@ def test_get_related_dataset(dataDir): ) # One related dataset md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)' - data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)' + data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) assert data[0]['id'] == 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb' - ncin.close() # Two related datasets md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.related_dataset = ( - 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent), ' - 'no.met:b7cb7934-78ca-4439-812e-f560df3fe7eb (auxiliary)') - data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.related_dataset = ( + 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent), ' + 'no.met:b7cb7934-78ca-4439-812e-f560df3fe7eb (auxiliary)') + data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) assert data[0]['id'] == 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb' assert data[1]['id'] == 'no.met:b7cb7934-78ca-4439-812e-f560df3fe7eb' - ncin.close() # Malformed relation md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb' - data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb' + data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) assert 'The global attribute "related_dataset" is malformed' in \ md.missing_attributes['errors'][0] - ncin.close() # Invalid relation type md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (child)' - data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (child)' + data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) assert 'The dataset relation type must be either' in \ md.missing_attributes['errors'][0] - ncin.close() # Invalid identifier pattern md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.related_dataset = 'b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)' - data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.related_dataset = 'b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)' + data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin) assert 'missing naming_authority in the identifier' in \ md.missing_attributes['errors'][0] - ncin.close() @pytest.mark.py_mmd_tools @@ -267,8 +264,8 @@ def testNc_to_mmd_Get_acdd_metadata(dataDir): key = "dataset_production_status" test_in = os.path.join(dataDir, "reference_nc.nc") md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - assert md.get_acdd_metadata(mmd_yaml[key], ncin, key) == "Complete" + with Dataset(test_in, "w", diskless=True) as ncin: + assert md.get_acdd_metadata(mmd_yaml[key], ncin, key) == "Complete" @pytest.mark.py_mmd_tools @@ -443,24 +440,23 @@ def test_get_operational_status(dataDir, monkeypatch): mmd_element = mmd_yaml["operational_status"] test_in = os.path.join(dataDir, "reference_nc.nc") md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - - # processing_level is not present - value = md.get_operational_status(mmd_element, ncin) - assert value == "Not available" - - # processing_level is not valid - ncin.processing_level = "kjhhas" - mmd_element["maxOccurs"] = "1" - value = md.get_operational_status(mmd_element, ncin) - assert ("The ACDD attribute 'processing_level' in MMD attribute 'operational_status'" - in md.missing_attributes['errors'][0]) + with Dataset(test_in, "w", diskless=True) as ncin: + # processing_level is not present + value = md.get_operational_status(mmd_element, ncin) + assert value == "Not available" - # repetition of processing_level is allowed - mmd_element["maxOccurs"] = "unbounded" - with pytest.raises(ValueError) as ve: + # processing_level is not valid + ncin.processing_level = "kjhhas" + mmd_element["maxOccurs"] = "1" value = md.get_operational_status(mmd_element, ncin) - assert str(ve.value) == "This is not expected..." + assert ("The ACDD attribute 'processing_level' in MMD attribute 'operational_status'" + in md.missing_attributes['errors'][0]) + + # repetition of processing_level is allowed + mmd_element["maxOccurs"] = "unbounded" + with pytest.raises(ValueError) as ve: + value = md.get_operational_status(mmd_element, ncin) + assert str(ve.value) == "This is not expected..." @pytest.mark.py_mmd_tools @@ -471,24 +467,23 @@ def test_dataset_production_status(dataDir): mmd_element = mmd_yaml["dataset_production_status"] test_in = os.path.join(dataDir, "reference_nc.nc") md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - - # dataset_production_status is not present - value = md.get_dataset_production_status(mmd_element, ncin) - assert value == "Complete" - - # dataset_production_status is not valid - ncin.dataset_production_status = "kjhhas" - mmd_element["maxOccurs"] = "1" - value = md.get_dataset_production_status(mmd_element, ncin) - assert "The ACDD attribute 'dataset_production_status'" - "must " in md.missing_attributes['errors'][0] + with Dataset(test_in, "w", diskless=True) as ncin: + # dataset_production_status is not present + value = md.get_dataset_production_status(mmd_element, ncin) + assert value == "Complete" - # repetition of processing_level is allowed - mmd_element["maxOccurs"] = "unbounded" - with pytest.raises(ValueError) as ve: + # dataset_production_status is not valid + ncin.dataset_production_status = "kjhhas" + mmd_element["maxOccurs"] = "1" value = md.get_dataset_production_status(mmd_element, ncin) - assert str(ve.value) == "This is not expected..." + assert "The ACDD attribute 'dataset_production_status'" + "must " in md.missing_attributes['errors'][0] + + # repetition of processing_level is allowed + mmd_element["maxOccurs"] = "unbounded" + with pytest.raises(ValueError) as ve: + value = md.get_dataset_production_status(mmd_element, ncin) + assert str(ve.value) == "This is not expected..." @pytest.mark.py_mmd_tools @@ -499,19 +494,18 @@ def test_get_quality_control(dataDir): mmd_element = mmd_yaml["quality_control"] test_in = os.path.join(dataDir, "reference_nc.nc") md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - - # processing_level is not valid - ncin.quality_control = "kjhhas" - mmd_element["maxOccurs"] = "1" - md.get_quality_control(mmd_element, ncin) - assert "The ACDD attribute 'quality_control' must " in md.missing_attributes['errors'][0] - - # repetition of processing_level is allowed - mmd_element["maxOccurs"] = "unbounded" - with pytest.raises(ValueError) as ve: + with Dataset(test_in, "w", diskless=True) as ncin: + # processing_level is not valid + ncin.quality_control = "kjhhas" + mmd_element["maxOccurs"] = "1" md.get_quality_control(mmd_element, ncin) - assert str(ve.value) == "This is not expected..." + assert "The ACDD attribute 'quality_control' must " in md.missing_attributes['errors'][0] + + # repetition of processing_level is allowed + mmd_element["maxOccurs"] = "unbounded" + with pytest.raises(ValueError) as ve: + md.get_quality_control(mmd_element, ncin) + assert str(ve.value) == "This is not expected..." @pytest.mark.py_mmd_tools @@ -947,10 +941,10 @@ def test_license__deprecated_attrs(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "CC-BY-4.0" - ncin.license_resource = "http://spdx.org/licenses/CC-BY-4.0" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "CC-BY-4.0" + ncin.license_resource = "http://spdx.org/licenses/CC-BY-4.0" + value = md.get_license(mmd_yaml['use_constraint'], ncin) self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0') self.assertEqual(value['identifier'], 'CC-BY-4.0') self.assertEqual( @@ -962,9 +956,9 @@ def test_license__invalid_url(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "spdx.org/licenses/CC-BY-4.0" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "spdx.org/licenses/CC-BY-4.0" + value = md.get_license(mmd_yaml['use_constraint'], ncin) self.assertIsNone(value) self.assertEqual( md.missing_attributes["errors"][0], @@ -979,9 +973,9 @@ def test_license__basic(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)" + value = md.get_license(mmd_yaml['use_constraint'], ncin) self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0') self.assertEqual(value['identifier'], 'CC-BY-4.0') @@ -992,9 +986,10 @@ def test_license__simple(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "http://spdx.org/licenses/CC-BY-4.0" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "http://spdx.org/licenses/CC-BY-4.0" + value = md.get_license(mmd_yaml['use_constraint'], ncin) + # met-vocab-tools should be able to find the license by # searching a url (see issue https://github.com/metno/met-vocab-tools/issues/25): # self.assertEqual(value['identifier'], 'CC-BY-4.0') @@ -1009,9 +1004,10 @@ def test_license__only_url_but_not_standard(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "http://spdx.org/licenses/CC-BY-4.1" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "http://spdx.org/licenses/CC-BY-4.1" + value = md.get_license(mmd_yaml['use_constraint'], ncin) + # met-vocab-tools should be able to find the license by # searching a url (see issue https://github.com/metno/met-vocab-tools/issues/25): # self.assertEqual(value['identifier'], 'CC-BY-4.0') @@ -1027,9 +1023,10 @@ def test_license__according_to_adc1(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "http://spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "http://spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)" + value = md.get_license(mmd_yaml['use_constraint'], ncin) + self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0') self.assertEqual(value['identifier'], 'CC-BY-4.0') @@ -1040,9 +1037,10 @@ def test_license__according_to_adc2(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)" + value = md.get_license(mmd_yaml['use_constraint'], ncin) + self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0') self.assertEqual(value['identifier'], 'CC-BY-4.0') @@ -1054,10 +1052,11 @@ def test_license__not_standard(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.reference_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.license = "https://earth.esa.int/eogateway/documents/20142/1564626/" \ - "ESA-Data-Policy-ESA-PB-EO-2010-54.pdf (ESA earth observation data policy)" - value = md.get_license(mmd_yaml['use_constraint'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.license = ("https://earth.esa.int/eogateway/documents/20142/1564626/" + "ESA-Data-Policy-ESA-PB-EO-2010-54.pdf (ESA earth " + "observation data policy)") + value = md.get_license(mmd_yaml['use_constraint'], ncin) self.assertEqual( value['license_text'], "https://earth.esa.int/eogateway/documents/20142/1564626/" @@ -1133,11 +1132,11 @@ def test_polygon_is_not_wkt(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.geospatial_bounds = "" - md.get_geographic_extent_polygon( - mmd_yaml['geographic_extent']['polygon'], ncin - ) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.geospatial_bounds = "" + md.get_geographic_extent_polygon( + mmd_yaml['geographic_extent']['polygon'], ncin + ) self.assertEqual( md.missing_attributes["errors"][0], "geospatial_bounds must be formatted as a WKT string" @@ -1242,13 +1241,13 @@ def test_geographic_extent_rectangle_is_floatable(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.geospatial_lat_max = "60.158733f; // float" - ncin.geospatial_lat_min = "59.78492f; // float" - ncin.geospatial_lon_max = "10.944986f; // float" - ncin.geospatial_lon_min = "10.508897f; // float" - md.get_geographic_extent_rectangle( - mmd_yaml['geographic_extent']['rectangle'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.geospatial_lat_max = "60.158733f; // float" + ncin.geospatial_lat_min = "59.78492f; // float" + ncin.geospatial_lon_max = "10.944986f; // float" + ncin.geospatial_lon_min = "10.508897f; // float" + md.get_geographic_extent_rectangle( + mmd_yaml['geographic_extent']['rectangle'], ncin) self.assertEqual( md.missing_attributes['errors'][0], 'geospatial_lat_max must be convertible to float type.') @@ -1398,12 +1397,9 @@ def test_alternate_identifier_wrong_format(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_with_altID.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.alternate_identifier = 'wrong format, missing type' - value = md.get_alternate_identifier( - mmd_yaml['alternate_identifier'], ncin - ) - print(value) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.alternate_identifier = 'wrong format, missing type' + md.get_alternate_identifier(mmd_yaml['alternate_identifier'], ncin) self.assertEqual( md.missing_attributes['errors'][0], 'alternate_identifier must be formed as ().' @@ -1558,16 +1554,15 @@ def test_temporal_extent_two_startdates_one_wrong(self): ) md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_attrs_multiple.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - valid_start = '2020-11-27T13:40:02.019817Z' invalid_start = '2020-13-27T13:40:02.019817' # month outside [0,12] valid_end = '2020-11-27T13:51:24.401505Z' invalid_end = '2020-13-27T13:51:24.019817' # month outside [0,12] - ncin.time_coverage_start = '{}, {}'.format(valid_start, invalid_start) - ncin.time_coverage_end = '{}, {}'.format(valid_end, invalid_end) - value = md.get_temporal_extents(mmd_yaml['temporal_extent'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.time_coverage_start = '{}, {}'.format(valid_start, invalid_start) + ncin.time_coverage_end = '{}, {}'.format(valid_end, invalid_end) + value = md.get_temporal_extents(mmd_yaml['temporal_extent'], ncin) template = 'ACDD start/end datetime {} is not valid ISO8601:' self.assertEqual(value[0]['start_date'], valid_start) @@ -1752,11 +1747,10 @@ def test_get_iso_topic_category_invalid(self): mmd_element = mmd_yaml["iso_topic_category"] test_in = os.path.abspath('tests/data/reference_nc.nc') md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - - # iso_topic_category is not valid - ncin.iso_topic_category = "abcd" - md.get_iso_topic_category(mmd_element, ncin) + with Dataset(test_in, "w", diskless=True) as ncin: + # iso_topic_category is not valid + ncin.iso_topic_category = "abcd" + md.get_iso_topic_category(mmd_element, ncin) assert "The ACDD attribute 'iso_topic_category' must" in md.missing_attributes['errors'][0] def test_get_iso_topic_category_not_available(self): @@ -1766,10 +1760,9 @@ def test_get_iso_topic_category_not_available(self): mmd_element = mmd_yaml["iso_topic_category"] test_in = os.path.abspath('tests/data/reference_nc.nc') md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - - # iso_topic_category is not present - value = md.get_iso_topic_category(mmd_element, ncin) + with Dataset(test_in, "w", diskless=True) as ncin: + # iso_topic_category is not present + value = md.get_iso_topic_category(mmd_element, ncin) assert value == ["Not available"] def test_get_activity_type_invalid(self): @@ -1782,11 +1775,10 @@ def test_get_activity_type_invalid(self): mmd_element = mmd_yaml["activity_type"] test_in = os.path.abspath('tests/data/reference_nc.nc') md = Nc_to_mmd(test_in, check_only=True) - ncin = Dataset(test_in, "w", diskless=True) - - # activity_type is not valid - ncin.source = "abcd" - md.get_activity_type(mmd_element, ncin) + with Dataset(test_in, "w", diskless=True) as ncin: + # activity_type is not valid + ncin.source = "abcd" + md.get_activity_type(mmd_element, ncin) assert ("The ACDD attribute 'source' in MMD attribute 'activity_type'" in md.missing_attributes['errors'][0]) @@ -1803,12 +1795,12 @@ def test_platform_resource_not_MMD(self): # The GCMD resource url resource = "https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/platforms" # Define dataset - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = "Sentinel-1A" - ncin.platform_vocabulary = resource + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = "Sentinel-1A" + ncin.platform_vocabulary = resource - # Get the platform dict - value = md.get_platforms(mmd_yaml['platform'], ncin) + # Get the platform dict + value = md.get_platforms(mmd_yaml['platform'], ncin) self.assertEqual(value[0]['resource'], resource) def test_missing_vocabulary_platform_instrument_short_name(self): @@ -1822,13 +1814,13 @@ def test_missing_vocabulary_platform_instrument_short_name(self): md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_missing_keywords_vocab.nc'), check_only=True) # Define dataset - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = 'Suomi National Polar-orbiting Partnership (SNPP)' - ncin.instrument = 'VIIRS' - ncin.instrument_vocabulary = 'not a valid vocab url' - ncin.platform_vocabulary = 'https://www.wmo-sat.info/oscar/satellites/view/342' + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = 'Suomi National Polar-orbiting Partnership (SNPP)' + ncin.instrument = 'VIIRS' + ncin.instrument_vocabulary = 'not a valid vocab url' + ncin.platform_vocabulary = 'https://www.wmo-sat.info/oscar/satellites/view/342' + value = md.get_platforms(mmd_yaml['platform'], ncin) - value = md.get_platforms(mmd_yaml['platform'], ncin) self.assertEqual(value[0]['resource'], 'https://www.wmo-sat.info/oscar/satellites/view/342') self.assertEqual(value[0]['short_name'], 'SNPP') @@ -1845,10 +1837,11 @@ def test_platform_vocabulary_invalid_url(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = 'Fake Environmental Satellite' - ncin.platform_vocabulary = 'invalid_url' - value = md.get_platforms(mmd_yaml['platform'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = 'Fake Environmental Satellite' + ncin.platform_vocabulary = 'invalid_url' + value = md.get_platforms(mmd_yaml['platform'], ncin) + self.assertEqual(value, []) self.assertEqual( md.missing_attributes['errors'][0], @@ -1867,9 +1860,9 @@ def test_wrong_platform_name(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(Too Many)(SNPP)' - value = md.get_platforms(mmd_yaml['platform'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(Too Many)(SNPP)' + value = md.get_platforms(mmd_yaml['platform'], ncin) self.assertEqual(value, []) self.assertEqual( md.missing_attributes['errors'][0], @@ -1887,10 +1880,11 @@ def test_wrong_instrument_name(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = "Suomi National Polar-orbiting Partnership (SNPP)" - ncin.instrument = 'InstrName (Instrument Name)(Too Many)(IN)' - value = md.get_platforms(mmd_yaml['platform'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = "Suomi National Polar-orbiting Partnership (SNPP)" + ncin.instrument = 'InstrName (Instrument Name)(Too Many)(IN)' + value = md.get_platforms(mmd_yaml['platform'], ncin) + self.assertEqual(value[0]['short_name'], "SNPP") self.assertEqual( md.missing_attributes['warnings'][0], @@ -1909,11 +1903,12 @@ def test_platform_name_extra_parentheses(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(SNPP)' - # A valid url is required - ncin.platform_vocabulary = "https://www.validurl.no" - value = md.get_platforms(mmd_yaml['platform'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(SNPP)' + # A valid url is required + ncin.platform_vocabulary = "https://www.validurl.no" + value = md.get_platforms(mmd_yaml['platform'], ncin) + self.assertEqual(value[0]['long_name'], 'Suomi National Polar-orbiting Partnership (Suomi NPP)') self.assertEqual(value[0]['short_name'], 'SNPP') @@ -1925,12 +1920,13 @@ def test_instrument_name_extra_parentheses(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.platform = "Platform Name (PN)" - ncin.platform_vocabulary = "https://www.validurl.no" - # This is an entry in MMD: - ncin.instrument = 'Synthetic Aperture Radar (C-band) (SAR-C)' - value = md.get_platforms(mmd_yaml['platform'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = "Platform Name (PN)" + ncin.platform_vocabulary = "https://www.validurl.no" + # This is an entry in MMD: + ncin.instrument = 'Synthetic Aperture Radar (C-band) (SAR-C)' + value = md.get_platforms(mmd_yaml['platform'], ncin) + self.assertEqual(value[0]['instrument']['short_name'], 'SAR-C') self.assertEqual(value[0]['instrument']['long_name'], 'Synthetic Aperture Radar (C-band)') @@ -1943,10 +1939,10 @@ def test_missing_platform_vocabulary(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - # Note that Envisat is not in the MMD vocabulary - ncin.platform = 'FakeEnvisat' - value = md.get_platforms(mmd_yaml['platform'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.platform = 'FakeEnvisat' + value = md.get_platforms(mmd_yaml['platform'], ncin) + self.assertEqual(value, []) self.assertEqual( md.missing_attributes['errors'][0], @@ -2015,13 +2011,14 @@ def test__keywords_vocabulary__correctly_formatted(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \ - "GEMET:Meteorological geographical features, " \ - "GEMET:Atmospheric conditions, " \ - "NORTHEMES:Weather and climate" - ncin.keywords_vocabulary = "" - md.get_keywords(mmd_yaml['keywords'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \ + "GEMET:Meteorological geographical features, " \ + "GEMET:Atmospheric conditions, " \ + "NORTHEMES:Weather and climate" + ncin.keywords_vocabulary = "" + md.get_keywords(mmd_yaml['keywords'], ncin) + self.assertEqual( md.missing_attributes['errors'][0], 'keywords_vocabulary must be formatted as ::' @@ -2035,18 +2032,19 @@ def test_keywords_vocabulary__invalid_url_pattern(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \ - "GEMET:Meteorological geographical features, " \ - "GEMET:Atmospheric conditions, " \ - "NORTHEMES:Weather and climate" - ncin.keywords_vocabulary = ( - "GCMDSK:GCMD Science Keywords:" - "https://gcmd.earth_data.nasa.gov/kms/concepts/concept_scheme/sciencekeywords, " - "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, " - "NORTHEMES:GeoNorge Themes:" - "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling") - md.get_keywords(mmd_yaml['keywords'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \ + "GEMET:Meteorological geographical features, " \ + "GEMET:Atmospheric conditions, " \ + "NORTHEMES:Weather and climate" + ncin.keywords_vocabulary = ( + "GCMDSK:GCMD Science Keywords:" + "https://gcmd.earth_data.nasa.gov/kms/concepts/concept_scheme/sciencekeywords, " + "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, " + "NORTHEMES:GeoNorge Themes:" + "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling") + md.get_keywords(mmd_yaml['keywords'], ncin) + self.assertEqual( md.missing_attributes['errors'][0], 'https://gcmd.earth_data.nasa.gov/kms/concepts/concept_scheme/sciencekeywords' @@ -2063,17 +2061,18 @@ def test_spaces_around_keywords_are_stripped(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.keywords = "GCMDSK: Earth Science > Atmosphere > Atmospheric radiation , " \ - "GEMET: Meteorological geographical features, " \ - "GEMET: Atmospheric conditions , " \ - "NORTHEMES:Weather and climate " - ncin.keywords_vocabulary = ( - "GCMDSK:GCMD Science Keywords:https://vocab.met.no/GCMDSK," - "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, " - "NORTHEMES:GeoNorge Themes:" - "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling") - data = md.get_keywords(mmd_yaml['keywords'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.keywords = "GCMDSK: Earth Science > Atmosphere > Atmospheric radiation , " \ + "GEMET: Meteorological geographical features, " \ + "GEMET: Atmospheric conditions , " \ + "NORTHEMES:Weather and climate " + ncin.keywords_vocabulary = ( + "GCMDSK:GCMD Science Keywords:https://vocab.met.no/GCMDSK," + "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, " + "NORTHEMES:GeoNorge Themes:" + "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling") + data = md.get_keywords(mmd_yaml['keywords'], ncin) + self.assertEqual(data[0]["keyword"][0], "Earth Science > Atmosphere > Atmospheric radiation") @@ -2269,13 +2268,15 @@ def test_dataset_citation_as_kwarg(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.creator_name = "Tester Test" - ncin.date_created = "2022-11-04T11:06:10Z" - ncin.title = "Test dataset" - ncin.publisher_name = "Norwegian Meteorological Institute" - ncin.metadata_link = "invalid_url" - value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin, dataset_citation=dc) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.creator_name = "Tester Test" + ncin.date_created = "2022-11-04T11:06:10Z" + ncin.title = "Test dataset" + ncin.publisher_name = "Norwegian Meteorological Institute" + ncin.metadata_link = "invalid_url" + value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin, + dataset_citation=dc) + self.assertEqual(value[0]['author'], 'No Name') def test_check_only(self): @@ -2322,24 +2323,26 @@ def test_dataset_citation_invalid_date(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc.nc'), check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.time_coverage_start = "2020-11-27T13:40:02.019817Z" - ncin.time_coverage_end = "2020-11-27T13:51:24.401505Z" - ncin.creator_name = 'Kreator Kreatorsen' - ncin.date_created = "2020-11-28T13:51:24.401505Z" - ncin.title = "Test dataset" - ncin.metadata_link = "https://data.met.no/dataset/uuid-for-the-dataset" - ncin.references = "Some free-text refences" - value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin) - self.assertEqual(value[0]['author'], 'Kreator Kreatorsen') + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.time_coverage_start = "2020-11-27T13:40:02.019817Z" + ncin.time_coverage_end = "2020-11-27T13:51:24.401505Z" + ncin.creator_name = 'Kreator Kreatorsen' + ncin.date_created = "2020-11-28T13:51:24.401505Z" + ncin.title = "Test dataset" + ncin.metadata_link = "https://data.met.no/dataset/uuid-for-the-dataset" + ncin.references = "Some free-text refences" + value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin) + + self.assertEqual(value[0]['author'], 'Kreator Kreatorsen') + + # Test that an error is appended if date created is not in + # the correct format + ncin.date_created = "2020-99-28 13:51:24" + mmd_yaml = yaml.load( + resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader + ) + value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin) - # Test that an error is appended if date created is not in - # the correct format - ncin.date_created = "2020-99-28 13:51:24" - mmd_yaml = yaml.load( - resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader - ) - value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin) self.assertIn( "ACDD attribute date_created contains an invalid ISO8601 date:", md.missing_attributes['errors'][0] @@ -2614,23 +2617,6 @@ def test_create_mmd_missing_update_times(self): 'ACDD attribute date_created is required' ) - def test_publication_date__is_a_list_of_dates(self): - """The publication data can be a list of dates but is set to - date_created. In most cases it is the only one item, but we - need to check that what we get from the netcdf file is an - actual list, and that the items are actual datestrings. - """ - mmd_yaml = yaml.load( - resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader - ) - md = Nc_to_mmd(self.reference_nc, check_only=True) - # To overwrite date_created, wihtout saving it to file we use diskless - ncin = Dataset(md.netcdf_file) - data = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin) - self.assertEqual( - data[0]['title'], - 'Direct Broadcast data processed in satellite swath to L1C.') - def test_get_metadata_updates__datetimes_not_iso(self): """ Test that an error is raised if datetimes of metadata updates are not ISO 8601. @@ -2640,9 +2626,10 @@ def test_get_metadata_updates__datetimes_not_iso(self): ) md = Nc_to_mmd(self.fail_nc, check_only=True) # To overwrite date_created, wihtout saving it to file we use diskless - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.date_created = '2019/01/01 00:00:00' - md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.date_created = '2019/01/01 00:00:00' + md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin) + self.assertEqual( md.missing_attributes['errors'][0], "Datetime element must be in ISO8601 format: " @@ -2656,13 +2643,14 @@ def test_ACDD_attr__date_created(self): ) md = Nc_to_mmd(self.fail_nc, check_only=True) # To overwrite date_created, wihtout saving it to file we use diskless - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.date_created = '2019-01-01T00:00:00Z' - data = md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin) - self.assertIn( - {'datetime': ncin.date_created, 'type': 'Created'}, - data['update'] - ) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.date_created = '2019-01-01T00:00:00Z' + data = md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin) + + self.assertIn( + {'datetime': ncin.date_created, 'type': 'Created'}, + data['update'] + ) assert (len(data['update']) == 1) self.assertEqual(data['update'][0]['type'], 'Created') @@ -2763,10 +2751,11 @@ def test_access_constraint(self): def test_check_attributes_not_empty(self): md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.geospatial_bounds = "" - with self.assertRaises(ValueError) as e: - md.check_attributes_not_empty(ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.geospatial_bounds = "" + with self.assertRaises(ValueError) as e: + md.check_attributes_not_empty(ncin) + self.assertIn('Global attribute geospatial_bounds is empty - please correct.', str(e.exception)) @@ -2776,18 +2765,19 @@ def test_check_attributes_not_empty__accepts_0(self): a bug. This test checks that 0 is accepted. """ md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.subswath = 0 - self.assertEqual(None, md.check_attributes_not_empty(ncin)) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.subswath = 0 + self.assertEqual(None, md.check_attributes_not_empty(ncin)) def test_check_feature_type__missing(self): """ Test that the correct warning is issued if the CF attribute featureType is missing. """ md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.tull = "tull" - md.check_feature_type(ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.tull = "tull" + md.check_feature_type(ncin) + self.assertEqual( md.missing_attributes["warnings"][0], "CF attribute featureType is missing - one of the feature" @@ -2800,9 +2790,10 @@ def test_check_feature_type__wrong(self): attribute featureType is wrong. """ md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.featureType = "tull" - md.check_feature_type(ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.featureType = "tull" + md.check_feature_type(ncin) + self.assertEqual( md.missing_attributes["errors"][0], "featureType seems to be wrong - it should be picked from " @@ -2812,9 +2803,10 @@ def test_check_feature_type__wrong(self): def test_check_conventions__missing(self): md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.tull = "tull" - md.check_conventions(ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.tull = "tull" + md.check_conventions(ncin) + self.assertEqual( md.missing_attributes["errors"][0], 'Required attribute "Conventions" is missing. This should' @@ -2823,9 +2815,10 @@ def test_check_conventions__missing(self): def test_check_conventions__cf_and_acdd_missing(self): md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.Conventions = "tull" - md.check_conventions(ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.Conventions = "tull" + md.check_conventions(ncin) + self.assertEqual( md.missing_attributes["errors"][0], 'The dataset should follow the CF-standard. Please ' @@ -2845,9 +2838,10 @@ def test_institution_name_parsing(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.institution = "Norwegian Meteorological Institute (MET Norway)" - data = md.get_data_centers(mmd_yaml['data_center'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.institution = "Norwegian Meteorological Institute (MET Norway)" + data = md.get_data_centers(mmd_yaml['data_center'], ncin) + self.assertEqual(data[0]['data_center_name']['long_name'], 'Norwegian Meteorological Institute') self.assertEqual(data[0]['data_center_name']['short_name'], 'MET Norway') @@ -2858,9 +2852,10 @@ def test_institution_short_name_missing(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.institution = "Norwegian Meteorological Institute" - md.get_data_centers(mmd_yaml['data_center'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.institution = "Norwegian Meteorological Institute" + md.get_data_centers(mmd_yaml['data_center'], ncin) + self.assertEqual( md.missing_attributes['errors'][0], "institution must be formed as (). " @@ -2887,13 +2882,15 @@ def test_acdd_references_as_related_information2(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.references = ( - "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01" - " (Dataset landing page)," # added a space - "https://ieeexplore.ieee.org/document/7914752 (Scientific publication)" # added space - ) - data = md.get_related_information(mmd_yaml['related_information'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.references = ( + "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01" + " (Dataset landing page)," # added a space + "https://ieeexplore.ieee.org/document/7914752 (Scientific " + "publication)" # added space + ) + data = md.get_related_information(mmd_yaml['related_information'], ncin) + # Note: A landing page url created from the netcdf id and # naming_authority attributes overrides the references information self.assertEqual( @@ -2912,13 +2909,14 @@ def test_acdd_references_invalid_type(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.references = ( - "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01" - "(kjhf)," - "https://ieeexplore.ieee.org/document/7914752(Scientific publication)" - ) - data = md.get_related_information(mmd_yaml['related_information'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.references = ( + "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01" + "(kjhf)," + "https://ieeexplore.ieee.org/document/7914752(Scientific publication)" + ) + data = md.get_related_information(mmd_yaml['related_information'], ncin) + self.assertEqual(data[1]['type'], 'Scientific publication') self.assertEqual( md.missing_attributes["errors"][0], @@ -2935,13 +2933,14 @@ def test_acdd_references_invalid_url(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.references = ( - "https://data.invalid_domain.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01" - "(Dataset landing page)," - "https://ieeexplore.ieee__.org/document/7914752(Scientific publication)" - ) - md.get_related_information(mmd_yaml['related_information'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.references = ( + "https://data.invalid_domain.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01" + "(Dataset landing page)," + "https://ieeexplore.ieee__.org/document/7914752(Scientific publication)" + ) + md.get_related_information(mmd_yaml['related_information'], ncin) + self.assertEqual( md.missing_attributes["errors"][0], 'references must contain valid uris') @@ -2957,9 +2956,10 @@ def test_acdd_references_malformed(self): resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader ) md = Nc_to_mmd(self.fail_nc, check_only=True) - ncin = Dataset(md.netcdf_file, "w", diskless=True) - ncin.references = "landing_page, paper" - md.get_related_information(mmd_yaml['related_information'], ncin) + with Dataset(md.netcdf_file, "w", diskless=True) as ncin: + ncin.references = "landing_page, paper" + md.get_related_information(mmd_yaml['related_information'], ncin) + self.assertEqual( md.missing_attributes["errors"][0], "references must be formed as ().")