From 87581eb3f83408263114385c23972f907185bc72 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Fri, 26 Jul 2024 17:18:54 +0200
Subject: [PATCH 01/32] #337: add code for moving data

---
 py_mmd_tools/mmd_operations.py   | 125 +++++++++++++++++++++++++++
 py_mmd_tools/script/move_data.py |  74 ++++++++++++++++
 tests/test_mmd_operations.py     | 139 +++++++++++++++++++++++++++++++
 tests/test_move_data_script.py   |  65 +++++++++++++++
 4 files changed, 403 insertions(+)
 create mode 100644 py_mmd_tools/mmd_operations.py
 create mode 100644 py_mmd_tools/script/move_data.py
 create mode 100644 tests/test_mmd_operations.py
 create mode 100644 tests/test_move_data_script.py

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
new file mode 100644
index 00000000..3676bf60
--- /dev/null
+++ b/py_mmd_tools/mmd_operations.py
@@ -0,0 +1,125 @@
+"""
+License:
+
+This file is part of the py-mmd-tools repository
+<https://github.com/metno/py-mmd-tools>.
+
+py-mmd-tools is licensed under the Apache License 2.0
+<https://github.com/metno/py-mmd-tools/blob/master/LICENSE>
+"""
+import os
+import re
+import pytz
+import uuid
+import shutil
+import netCDF4
+import datetime
+import tempfile
+import datetime_glob
+
+
+def add_metadata_update_info(f, note, type="Minor modification"):
+    """ Add update information """
+    f.write("    <mmd:update>\n"
+            "      <mmd:datetime>%s</mmd:datetime>\n"
+            "      <mmd:type>%s</mmd:type>\n"
+            "      <mmd:note>%s</mmd:note>\n"
+            "    </mmd:update>\n" % (datetime.datetime.utcnow().replace(
+        tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), type, note))
+
+
+def get_local_mmd_git_path(nc_file, mmd_repository_path):
+    """Return the path to the original MMD file.
+    """
+    ds = netCDF4.Dataset(nc_file)
+    lvlA = "arch_%s" % uuid.UUID(ds.id).hex[7]
+    lvlB = "arch_%s" % uuid.UUID(ds.id).hex[6]
+    lvlC = "arch_%s" % uuid.UUID(ds.id).hex[5]
+    mmd_filename = ds.id + ".xml"
+    ds.close()
+    return os.path.join(mmd_repository_path, lvlA, lvlB, lvlC, mmd_filename)
+
+
+def mmd_change_file_location(mmd, new_file_location, copy=True):
+    """Copy original MMD file, and make changes. Return the filename
+    of the updated file, and a status flag indicating if it has been
+    changed or not.
+    """
+    if copy:
+        tmp_path = tempfile.gettempdir()
+        shutil.copy2(mmd, tmp_path)
+        # Edit copied MMD file
+        mmd = os.path.join(tmp_path, os.path.basename(mmd))
+    lines = mmd_readlines(mmd)
+    # Open the MMD file and add updates
+    status = False
+    with open(mmd, "w") as f:
+        for line in lines:
+            if "</mmd:last_metadata_update>" in line:
+                add_metadata_update_info(f, "New file location in storage information.")
+            if "<mmd:file_location>" in line:
+                f.write(f"    <mmd:file_location>{new_file_location}</mmd:file_location>\n")
+                status = True
+            else:
+                f.write(line)
+    return mmd, status
+
+
+def mmd_readlines(filename):
+    """ Read lines in MMD file.
+    """
+    if not os.path.exists(filename):
+        raise ValueError("File does not exist: %s" % filename)
+    with open(filename, "r") as f:
+        lines = f.readlines()
+    return lines
+
+
+def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pattern_or_exact):
+    """Update MMD and move data file.
+    """
+    if os.path.isfile(existing_pathname_pattern_or_exact):
+        existing = [existing_pathname_pattern_or_exact]
+        existing_pathname_pattern = None
+    else:
+        existing = [file for match, file in
+                    datetime_glob.walk(pattern=existing_pathname_pattern_or_exact)]
+        existing_pathname_pattern = existing_pathname_pattern_or_exact
+
+    updated = []
+    for file in existing:
+        nfl = new_file_location(file, new_file_location_base, existing_pathname_pattern)
+        mmd_orig = get_local_mmd_git_path(file, mmd_repository_path)
+        mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl)
+        if not mmd_updated:
+            raise Exception("MMD was not updated..")
+
+        # Update with dmci update
+        dmci_updated = True
+
+        # If update was ok - move netcdf file
+        # mv file nfl
+        nc_moved = True
+
+        # Check by searching CSW and checking data access urls
+        ds_found_and_accessible = True
+
+        updated.append(all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]))
+
+    return all(updated), mmd_new
+
+
+def new_file_location(file, new_base_loc, existing_pathname_pattern=None):
+    """Return the new file location. If existing_pathname_pattern is
+    None, the returned path will equal the provided parameter
+    new_base_loc. This is to allow usage flexibility of the move_data
+    function.
+    """
+    if existing_pathname_pattern == None:
+        file_path = os.path.join(new_base_loc, os.path.basename(file))
+    else:
+        file_path = os.path.join(new_base_loc, file.removeprefix(re.split(r"[^\w/-]",
+                                                                 existing_pathname_pattern)[0]))
+    if not os.path.isfile(file_path):
+        raise ValueError(f"File does not exist: {file_path}")
+    return os.path.dirname(os.path.abspath(file_path))
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
new file mode 100644
index 00000000..7444497f
--- /dev/null
+++ b/py_mmd_tools/script/move_data.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+Script to move one or more datasets from one location to another, and
+update its MMD xml file accordingly.
+
+License:
+
+This file is part of the py-mmd-tools repository
+<https://github.com/metno/py-mmd-tools>.
+
+py-mmd-tools is licensed under the Apache License 2.0
+<https://github.com/metno/py-mmd-tools/blob/master/LICENSE>
+
+Usage:
+    move_data [-h] -i INPUT -n OUTPUT_DIR
+"""
+import os
+import re
+import glob
+import uuid
+import netCDF4
+import pathlib
+import argparse
+import tempfile
+import datetime_glob
+
+from py_mmd_tools.mmd_operations import move_data
+from py_mmd_tools.mmd_operations import mmd_readlines
+from py_mmd_tools.mmd_operations import new_file_location
+from py_mmd_tools.mmd_operations import get_local_mmd_git_path
+from py_mmd_tools.mmd_operations import add_metadata_update_info
+from py_mmd_tools.mmd_operations import mmd_change_file_location
+
+
+def create_parser():
+    """Create parser object"""
+    parser = argparse.ArgumentParser(description="Move one or more datasets from one location to "
+                                                 "another, and update its MMD xml file "
+                                                 "accordingly.")
+    parser.add_argument(
+        "mmd_repository_path", type=str,
+        help="Local folder containing all MMD files.")
+    parser.add_argument(
+        "new_file_location_base", type=str,
+        help="Base or exact path to the new file location.")
+    parser.add_argument(
+        "existing_pathname_pattern", type=str,
+        help="Pathname pattern to existing file location(s). Allows "
+             "parsing date/times from a path given a glob pattern "
+             "intertwined with date/time format akin to "
+             "strptime/strftime format.")
+    return parser
+
+
+def main(args=None):
+    """Move dataset(s) and update MMD.
+    """
+    if not os.path.isdir(args.mmd_repository_path):
+        raise ValueError(f"Invalid input: {args.mmd_repository_path}")
+
+    if not os.path.isdir(args.new_file_location_base):
+        raise ValueError(f"Invalid input: {args.new_file_location_base}")
+
+    return move_data(args.mmd_repository_path,
+                     args.new_file_location_base,
+                     args.existing_pathname_pattern)
+
+
+def _main():  # pragma: no cover
+    main(create_parser().parse_args())  # entry point in setup.cfg
+
+
+if __name__ == '__main__':  # pragma: no cover
+    main(create_parser().parse_args())
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
new file mode 100644
index 00000000..18c18b95
--- /dev/null
+++ b/tests/test_mmd_operations.py
@@ -0,0 +1,139 @@
+"""
+License:
+
+This file is part of the py-mmd-tools repository
+<https://github.com/metno/py-mmd-tools>.
+
+py-mmd-tools is licensed under the Apache License 2.0
+<https://github.com/metno/py-mmd-tools/blob/master/LICENSE>
+"""
+import os
+import pytest
+import shutil
+
+from py_mmd_tools.mmd_operations import move_data
+from py_mmd_tools.mmd_operations import mmd_readlines
+from py_mmd_tools.mmd_operations import new_file_location
+from py_mmd_tools.mmd_operations import mmd_change_file_location
+from py_mmd_tools.mmd_operations import get_local_mmd_git_path
+
+
+@pytest.mark.script
+def test_get_local_mmd_git_path(dataDir):
+    """Test that the mmd git path is returned correctly with folders
+    composed from its uuid.
+    """
+    ncfile = os.path.join(dataDir, "reference_nc.nc")
+    fn = get_local_mmd_git_path(ncfile, "/some/folder/mmd-xml-production")
+    assert fn == "/some/folder/mmd-xml-production/arch_4/arch_3/arch_9/" \
+                 "b7cb7934-77ca-4439-812e-f560df3fe7eb.xml"
+
+@pytest.mark.script
+def test_mmd_change_file_location(dataDir):
+    """Test that an MMD file is created with new file_location, and
+    new metadata update.
+    """
+    mmd = os.path.join(dataDir, "reference_nc_TMP.xml")
+    shutil.copy(os.path.join(dataDir, "reference_nc.xml"), mmd)
+    new_file_location = "/some/where/else/2024/06/19"
+
+    new_mmd, changed = mmd_change_file_location(mmd, new_file_location)
+    assert changed == True
+    assert os.path.isfile(new_mmd)
+    lines = mmd_readlines(new_mmd)
+    for line in lines:
+        if "<mmd:file_location>" in line:
+            assert "/some/where/else/2024/06/19" in line
+
+    mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False)
+    assert changed == True
+    assert os.path.isfile(mmd)
+    lines = mmd_readlines(mmd)
+    for line in lines:
+        if "file_location" in line:
+            assert "/some/where/else/2024/06/19" in line
+
+    # Delete tmp files
+    os.remove(mmd)
+    os.remove(new_mmd)
+
+
+@pytest.mark.script
+def test_mmd_readlines(dataDir):
+    """Test that the lines in the MMD file are returned as a list.
+    """
+    mmd = os.path.join(dataDir, "reference_nc.xml")
+    lines = mmd_readlines(mmd)
+    assert isinstance(lines, list)
+    assert "mmd:mmd" in lines[0]
+
+    mmd = os.path.join(dataDir, "does_not_exist.xml")
+    with pytest.raises(ValueError):
+        lines = mmd_readlines(mmd)
+
+
+@pytest.mark.script
+def test_move_data(dataDir, monkeypatch):
+    """Test the move_data function.
+    """
+    mmd_repository_path = "/some/folder/mmd-xml-production"
+    new_file_location_base = "/some/where/new"
+    nc_file = os.path.join(dataDir, "reference_nc.nc")
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
+                   lambda *a, **k: new_file_location_base)
+        mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
+                   lambda *a, **k: os.path.join(dataDir, "reference_nc.xml"))
+        updated, mmd_new = move_data(mmd_repository_path, new_file_location_base, nc_file)
+        assert updated == True
+        assert os.path.isfile(mmd_new)
+        lines = mmd_readlines(mmd_new)
+        for line in lines:
+            if "<mmd:file_location>" in line:
+                assert "/some/where/new" in line
+        # Remove new MMD file
+        os.remove(mmd_new)
+
+        def mock_walk(*a, **k):
+            yield (1, nc_file)
+
+        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
+                   lambda *a, **k: mock_walk(*a, **k))
+        # This pattern is not really used but it should look something
+        # like this when used correctly:
+        pattern = os.path.join(dataDir, "%Y/%m/%d/*.nc")
+        updated, mmd_new = move_data(mmd_repository_path, new_file_location_base, pattern)
+        assert updated == True
+        assert os.path.isfile(mmd_new)
+        lines = mmd_readlines(mmd_new)
+        for line in lines:
+            if "<mmd:file_location>" in line:
+                assert "/some/where/new" in line
+        # Remove new MMD file
+        os.remove(mmd_new)
+
+        mp.setattr("py_mmd_tools.mmd_operations.mmd_change_file_location",
+                   lambda *a, **k: (nc_file, False))
+        with pytest.raises(Exception):
+            updated, mmd_new = move_data(mmd_repository_path, new_file_location_base, pattern)
+
+
+@pytest.mark.script
+def test_new_file_location(monkeypatch):
+    """Test that the returned new location paths are correct.
+    """
+    file = "/some/old/loc/2024/06/19/" \
+           "S1A_IW_GRDM_1SDV_20240619T053156_20240619T053223_054388_069E03_FC95_MEPS.nc"
+    new_base = "/some/where/else"
+    existing_pathname_pattern = "/some/old/loc/%Y/%m/%d/*.nc"
+
+    with pytest.raises(ValueError):
+        new_file_location(file, new_base, existing_pathname_pattern)
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.os.path.isfile", lambda *a, **k: True)
+        assert new_file_location(file, new_base, existing_pathname_pattern) == \
+            "/some/where/else/2024/06/19"
+        new_base = "/some/where/else/2024/06/19"
+        assert new_file_location(file, new_base) == "/some/where/else/2024/06/19"
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
new file mode 100644
index 00000000..e2c2c419
--- /dev/null
+++ b/tests/test_move_data_script.py
@@ -0,0 +1,65 @@
+"""
+License:
+
+This file is part of the py-mmd-tools repository
+<https://github.com/metno/py-mmd-tools>.
+
+py-mmd-tools is licensed under the Apache License 2.0
+<https://github.com/metno/py-mmd-tools/blob/master/LICENSE>
+"""
+import os
+import pytest
+
+from py_mmd_tools.mmd_operations import mmd_readlines
+
+from py_mmd_tools.script.move_data import main
+from py_mmd_tools.script.move_data import create_parser
+
+
+@pytest.mark.script
+def test_main(dataDir, monkeypatch):
+    """
+    """
+    mmd_repository_path = "/some/folder/mmd-xml-production"
+    new_file_location_base = "/some/where/new"
+    existing_pathname_pattern = os.path.join(dataDir, "reference_nc.nc")
+    parser = create_parser()
+    parsed = parser.parse_args([
+        mmd_repository_path,
+        new_file_location_base,
+        existing_pathname_pattern
+    ])
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",
+                   lambda *a, **k: True)
+        mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
+                   lambda *a, **k: new_file_location_base)
+        mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
+                   lambda *a, **k: os.path.join(dataDir, "reference_nc.xml"))
+        updated, mmd_new = main(parsed)
+        assert updated == True
+        assert os.path.isfile(mmd_new)
+        lines = mmd_readlines(mmd_new)
+        for line in lines:
+            if "<mmd:file_location>" in line:
+                assert "/some/where/new" in line
+
+        mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",
+                   lambda *a, **k: False)
+        with pytest.raises(ValueError):
+            updated, mmd_new = main(parsed)
+
+        map = {
+            mmd_repository_path: True,
+            new_file_location_base: False
+        }
+        def mock_isdir(pp):
+            return map[pp]
+
+        mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", mock_isdir)
+        with pytest.raises(ValueError):
+            updated, mmd_new = main(parsed)
+
+        # Remove new MMD file
+        os.remove(mmd_new)

From 996b5961d2a0cef44fb87985dcdff1b6784a2dc4 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 16:07:57 +0200
Subject: [PATCH 02/32] #337: update MMD file

---
 tests/data/reference_nc.xml | 48 +++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 15 deletions(-)

diff --git a/tests/data/reference_nc.xml b/tests/data/reference_nc.xml
index cdeb0bf6..72976499 100644
--- a/tests/data/reference_nc.xml
+++ b/tests/data/reference_nc.xml
@@ -6,14 +6,8 @@
   <mmd:abstract xml:lang="no">Norsk abstrakt.</mmd:abstract>
   <mmd:metadata_status>Active</mmd:metadata_status>
   <mmd:dataset_production_status>In Work</mmd:dataset_production_status>
-  <mmd:collection>ADC</mmd:collection>
   <mmd:collection>METNCS</mmd:collection>
   <mmd:last_metadata_update>
-    <mmd:update>
-      <mmd:datetime>2020-11-27T14:05:56Z</mmd:datetime>
-      <mmd:note/>
-      <mmd:type>Created</mmd:type>
-    </mmd:update>
     <mmd:update>
       <mmd:datetime>2020-11-27T14:05:56Z</mmd:datetime>
       <mmd:type>Created</mmd:type>
@@ -43,6 +37,11 @@
     <mmd:resource>https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling</mmd:resource>
     <mmd:separator></mmd:separator>
   </mmd:keywords>
+  <mmd:keywords vocabulary="CFSTDN">
+    <mmd:keyword>toa_bidirectional_reflectance</mmd:keyword>
+    <mmd:resource>https://vocab.met.no/mmd/Keywords_Vocabulary/CFSTDN</mmd:resource>
+    <mmd:separator></mmd:separator>
+  </mmd:keywords>
   <mmd:geographic_extent>
     <mmd:rectangle srsName="EPSG:4326">
       <mmd:north>77.96752166748047</mmd:north>
@@ -115,9 +114,6 @@
     <mmd:name>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:name>
     <mmd:email>post@met.no</mmd:email>
     <mmd:organisation>Norwegian Meteorological Institute</mmd:organisation>
-    <mmd:contact_address>
-        <mmd:country>NORWAY</mmd:country>
-    </mmd:contact_address>
   </mmd:personnel>
   <mmd:data_center>
     <mmd:data_center_name>
@@ -126,13 +122,34 @@
     </mmd:data_center_name>
     <mmd:data_center_url>met.no</mmd:data_center_url>
   </mmd:data_center>
+  <mmd:quality_control>No quality control</mmd:quality_control>
+  <mmd:data_access>
+    <mmd:type>OPeNDAP</mmd:type>
+    <mmd:description>Open-source Project for a Network Data Access Protocol</mmd:description>
+    <mmd:resource>https://thredds.met.no/thredds/dodsC/reference_nc.nc</mmd:resource>
+  </mmd:data_access>
+  <mmd:data_access>
+    <mmd:type>HTTP</mmd:type>
+    <mmd:description>Direct download of file</mmd:description>
+    <mmd:resource>https://thredds.met.no/thredds/fileServer/reference_nc.nc</mmd:resource>
+  </mmd:data_access>
   <mmd:storage_information>
     <mmd:file_name>reference_nc.nc</mmd:file_name>
-    <mmd:file_location>tests/data/reference_nc.nc</mmd:file_location>
+    <mmd:file_location>/home/mortenwh/dev/py-mmd-tools/tests/data</mmd:file_location>
     <mmd:file_format>NetCDF-CF</mmd:file_format>
-    <mmd:file_size unit="MB">0.02</mmd:file_size>
-    <mmd:checksum type=""></mmd:checksum>
+    <mmd:file_size unit="MB">0.03</mmd:file_size>
+    <mmd:checksum type="md5sum">aa188b45ece9bdbbc9b470106a6b13f8</mmd:checksum>
   </mmd:storage_information>
+  <mmd:related_information>
+    <mmd:type>Dataset landing page</mmd:type>
+    <mmd:description></mmd:description>
+    <mmd:resource>https://data.met.no/dataset/b7cb7934-77ca-4439-812e-f560df3fe7eb</mmd:resource>
+  </mmd:related_information>
+  <mmd:related_information>
+    <mmd:type>Scientific publication</mmd:type>
+    <mmd:description></mmd:description>
+    <mmd:resource>https://ieeexplore.ieee.org/document/7914752</mmd:resource>
+  </mmd:related_information>
   <mmd:project>
     <mmd:short_name>METNCS</mmd:short_name>
     <mmd:long_name>MET Norway core services</mmd:long_name>
@@ -148,11 +165,12 @@
     </mmd:instrument>
   </mmd:platform>
   <mmd:spatial_representation>grid</mmd:spatial_representation>
+  <mmd:activity_type>Not available</mmd:activity_type>
   <mmd:dataset_citation>
     <mmd:author>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:author>
     <mmd:publication_date>2020-11-27T14:05:56Z</mmd:publication_date>
     <mmd:title>Direct Broadcast data processed in satellite swath to L1C.</mmd:title>
-    <mmd:url>https://met.no/the/dataset/landing-page</mmd:url>
-    <mmd:other>https://met.no/published/or/web-based/references/that/describe/the/data/or/methods/used/to/produce/it/should/be/URI</mmd:other>
+    <mmd:publisher>Norwegian Meteorological Institute</mmd:publisher>
+    <mmd:url>https://data.met.no/dataset/b7cb7934-77ca-4439-812e-f560df3fe7eb</mmd:url>
   </mmd:dataset_citation>
-</mmd:mmd>
+</mmd:mmd>
\ No newline at end of file

From c31a81b8f384f5f554f97bf17e324169888c691c Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 17:08:52 +0200
Subject: [PATCH 03/32] add contact address, country

---
 tests/data/reference_nc.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/data/reference_nc.xml b/tests/data/reference_nc.xml
index 72976499..fe4a38d1 100644
--- a/tests/data/reference_nc.xml
+++ b/tests/data/reference_nc.xml
@@ -114,6 +114,9 @@
     <mmd:name>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:name>
     <mmd:email>post@met.no</mmd:email>
     <mmd:organisation>Norwegian Meteorological Institute</mmd:organisation>
+    <mmd:contact_address>
+        <mmd:country>NORWAY</mmd:country>
+    </mmd:contact_address>
   </mmd:personnel>
   <mmd:data_center>
     <mmd:data_center_name>
@@ -173,4 +176,4 @@
     <mmd:publisher>Norwegian Meteorological Institute</mmd:publisher>
     <mmd:url>https://data.met.no/dataset/b7cb7934-77ca-4439-812e-f560df3fe7eb</mmd:url>
   </mmd:dataset_citation>
-</mmd:mmd>
\ No newline at end of file
+</mmd:mmd>

From cae9f8dbeb6a86c703aea49d059270bd1fd711d2 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 17:10:09 +0200
Subject: [PATCH 04/32] #337: bug fix - see line 113, and return values of
 get_acdd

---
 py_mmd_tools/mmd_to_nc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/py_mmd_tools/mmd_to_nc.py b/py_mmd_tools/mmd_to_nc.py
index 0d52817d..fb74bac8 100644
--- a/py_mmd_tools/mmd_to_nc.py
+++ b/py_mmd_tools/mmd_to_nc.py
@@ -119,7 +119,7 @@ def process_element(self, xml_element, translations):
                     raise ValueError('Multiple ACDD or ACCD extension fields provided.'
                                      ' Please use another translation function.')
                 # Update the dictionary containing the ACDD elements
-                self.update_acdd({acdd_name[0]: xml_element.text}, {acdd_name[0]: sep})
+                self.update_acdd({acdd_name[0]: xml_element.text}, {acdd_name[0]: sep[0]})
 
     def update_acdd(self, new_dict, sep=None):
         """

From e9838c93c1c33c03576f809c2d13b7ac17c22fe0 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 18:13:19 +0200
Subject: [PATCH 05/32] #337: this should work..

---
 py_mmd_tools/mmd_operations.py   | 83 +++++++++++++++++++++++++++-----
 py_mmd_tools/script/move_data.py |  8 ++-
 tests/data/reference_nc.xml      |  6 ++-
 tests/test_mmd_operations.py     | 43 ++++++++++++-----
 tests/test_mmd_to_nc.py          |  2 +-
 tests/test_move_data_script.py   | 27 ++++++++---
 6 files changed, 136 insertions(+), 33 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 3676bf60..5200fe3c 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -14,6 +14,7 @@
 import shutil
 import netCDF4
 import datetime
+import requests
 import tempfile
 import datetime_glob
 
@@ -75,9 +76,34 @@ def mmd_readlines(filename):
     return lines
 
 
-def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pattern_or_exact):
+def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pattern_or_exact,
+              dry_run=True, env="prod"):
     """Update MMD and move data file.
     """
+    if env not in ["dev", "staging", "prod"]:
+        raise ValueError("Invalid env input")
+    if env not in mmd_repository_path:
+        raise ValueError("Invalid mmd_repository path")
+
+    urls = {
+        "prod": {
+            "dmci": "dmci.s-enda.k8s.met.no",
+            "csw": "data.csw.met.no",
+            "id_namespace": "no.met",
+        },
+        "staging": {
+            "dmci": "dmci.s-enda-staging.k8s.met.no",
+            "csw": "https://csw.s-enda-staging.k8s.met.no/",
+            "id_namespace": "no.met.staging",
+        },
+        "dev": {
+            "dmci": "dmci.s-enda-dev.k8s.met.no",
+            "csw": "https://csw.s-enda-dev.k8s.met.no/",
+            "id_namespace": "no.met.dev",
+        }
+    }
+
+
     if os.path.isfile(existing_pathname_pattern_or_exact):
         existing = [existing_pathname_pattern_or_exact]
         existing_pathname_pattern = None
@@ -86,27 +112,62 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
                     datetime_glob.walk(pattern=existing_pathname_pattern_or_exact)]
         existing_pathname_pattern = existing_pathname_pattern_or_exact
 
+    if dry_run:
+        # Not copying the file will make it easy to check changes
+        # with git diff
+        existing_pathname_pattern = None
+
     updated = []
+    not_updated = []
     for file in existing:
         nfl = new_file_location(file, new_file_location_base, existing_pathname_pattern)
         mmd_orig = get_local_mmd_git_path(file, mmd_repository_path)
         mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl)
-        if not mmd_updated:
-            raise Exception("MMD was not updated..")
+
+        # Get MMD content as binary data
+        with open(mmd_new, "rb") as fn:
+            data = fn.read()
+
+        res = requests.post(url=f"https://{urls[env]['dmci']}/v1/validate", data=data)
+        dmci_valid = False
+        if res.status_code == 200:
+            dmci_valid = True
+
 
         # Update with dmci update
-        dmci_updated = True
+        dmci_updated = False
+        if not dry_run:
+            # be careful with this...
+            res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data)
+        if res.status_code == 200:
+            # This intentionally becomes True in case of dry-run and
+            # a valid xml
+            dmci_updated = True
 
         # If update was ok - move netcdf file
-        # mv file nfl
-        nc_moved = True
+        nc_moved = False
+        if dmci_updated:
+            shutil.move(file, nfl)
+            nc_moved = True
 
         # Check by searching CSW and checking data access urls
-        ds_found_and_accessible = True
-
-        updated.append(all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]))
-
-    return all(updated), mmd_new
+        res = requests.get(url=f"https://{urls[env]['csw']}/csw",
+                           params={
+                               "service": "CSW",
+                               "version": "2.0.2",
+                               "request": "GetRepositoryItem",
+                               "id": f"no.met.{urls[env]['id_namespace']}"
+                                     f"{os.path.basename(file).split('.')[0]}"})
+        ds_found_and_accessible = False
+        if res.status_code == 200:
+            ds_found_and_accessible = True
+
+        if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]):
+            updated.append(mmd_new)
+        else:
+            not_updated.append(mmd_new)
+
+    return not_updated, updated
 
 
 def new_file_location(file, new_base_loc, existing_pathname_pattern=None):
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index 7444497f..d6e6663b 100644
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -49,6 +49,11 @@ def create_parser():
              "parsing date/times from a path given a glob pattern "
              "intertwined with date/time format akin to "
              "strptime/strftime format.")
+    parser.add_argument(
+        '--dmci-update', action='store_true',
+        help='Directly update the online catalog with the changed MMD files.'
+    )
+        
     return parser
 
 
@@ -63,7 +68,8 @@ def main(args=None):
 
     return move_data(args.mmd_repository_path,
                      args.new_file_location_base,
-                     args.existing_pathname_pattern)
+                     args.existing_pathname_pattern,
+                     dry_run=not args.dmci_update)
 
 
 def _main():  # pragma: no cover
diff --git a/tests/data/reference_nc.xml b/tests/data/reference_nc.xml
index fe4a38d1..9a4e79a1 100644
--- a/tests/data/reference_nc.xml
+++ b/tests/data/reference_nc.xml
@@ -115,7 +115,11 @@
     <mmd:email>post@met.no</mmd:email>
     <mmd:organisation>Norwegian Meteorological Institute</mmd:organisation>
     <mmd:contact_address>
-        <mmd:country>NORWAY</mmd:country>
+      <mmd:address>Meteorologisk institutt, Henrik Mohnsplass 1</mmd:address>
+      <mmd:city>Oslo</mmd:city>
+      <mmd:province_or_state>Oslo</mmd:province_or_state>
+      <mmd:postal_code>0000</mmd:postal_code>
+      <mmd:country>Norway</mmd:country>
     </mmd:contact_address>
   </mmd:personnel>
   <mmd:data_center>
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 18c18b95..43e21a4b 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -80,20 +80,37 @@ def test_move_data(dataDir, monkeypatch):
     new_file_location_base = "/some/where/new"
     nc_file = os.path.join(dataDir, "reference_nc.nc")
 
+    class MockResponse:
+
+        status_code = 200
+
+    with pytest.raises(ValueError):
+        move_data(mmd_repository_path, new_file_location_base, nc_file, env="hei")
+
+    with pytest.raises(ValueError):
+        move_data("/some/folder/mmd-xml-noenv", new_file_location_base, nc_file)
+
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
                    lambda *a, **k: new_file_location_base)
         mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
                    lambda *a, **k: os.path.join(dataDir, "reference_nc.xml"))
-        updated, mmd_new = move_data(mmd_repository_path, new_file_location_base, nc_file)
-        assert updated == True
-        assert os.path.isfile(mmd_new)
-        lines = mmd_readlines(mmd_new)
+        mp.setattr("py_mmd_tools.mmd_operations.shutil.move",
+                   lambda *a, **k: None)
+        mp.setattr("py_mmd_tools.mmd_operations.requests.get",
+                   lambda *a, **k: MockResponse())
+        mp.setattr("py_mmd_tools.mmd_operations.requests.post",
+                   lambda *a, **k: MockResponse())
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, nc_file)
+        assert len(not_updated) == 0
+        assert len(updated) == 1
+        assert os.path.isfile(updated[0])
+        lines = mmd_readlines(updated[0])
         for line in lines:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
         # Remove new MMD file
-        os.remove(mmd_new)
+        os.remove(updated[0])
 
         def mock_walk(*a, **k):
             yield (1, nc_file)
@@ -103,20 +120,22 @@ def mock_walk(*a, **k):
         # This pattern is not really used but it should look something
         # like this when used correctly:
         pattern = os.path.join(dataDir, "%Y/%m/%d/*.nc")
-        updated, mmd_new = move_data(mmd_repository_path, new_file_location_base, pattern)
-        assert updated == True
-        assert os.path.isfile(mmd_new)
-        lines = mmd_readlines(mmd_new)
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern,
+                                         dry_run=False)
+        assert len(not_updated) == 0
+        assert len(updated) == 1
+        assert os.path.isfile(updated[0])
+        lines = mmd_readlines(updated[0])
         for line in lines:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
         # Remove new MMD file
-        os.remove(mmd_new)
+        os.remove(updated[0])
 
         mp.setattr("py_mmd_tools.mmd_operations.mmd_change_file_location",
                    lambda *a, **k: (nc_file, False))
-        with pytest.raises(Exception):
-            updated, mmd_new = move_data(mmd_repository_path, new_file_location_base, pattern)
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+        assert len(not_updated) == 1
 
 
 @pytest.mark.script
diff --git a/tests/test_mmd_to_nc.py b/tests/test_mmd_to_nc.py
index 6553131f..6900710b 100644
--- a/tests/test_mmd_to_nc.py
+++ b/tests/test_mmd_to_nc.py
@@ -417,7 +417,7 @@ def test_process_element_2(self):
         md.process_element(element_to_translate, md.mmd_yaml)
         self.assertIsNone(md.acdd_metadata)
         # MMD element listed in the translation dictionary, but with no translation information
-        element_to_translate = md.tree.find('mmd:last_metadata_update/mmd:update/mmd:note',
+        element_to_translate = md.tree.find('mmd:last_metadata_update/mmd:update/mmd:type',
                                             md.namespaces)
         md.process_element(element_to_translate, md.mmd_yaml)
         self.assertIsNone(md.acdd_metadata)
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index e2c2c419..5aab9402 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -30,6 +30,10 @@ def test_main(dataDir, monkeypatch):
         existing_pathname_pattern
     ])
 
+    class MockResponse:
+
+        status_code = 200
+
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",
                    lambda *a, **k: True)
@@ -37,10 +41,17 @@ def test_main(dataDir, monkeypatch):
                    lambda *a, **k: new_file_location_base)
         mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
                    lambda *a, **k: os.path.join(dataDir, "reference_nc.xml"))
-        updated, mmd_new = main(parsed)
-        assert updated == True
-        assert os.path.isfile(mmd_new)
-        lines = mmd_readlines(mmd_new)
+        mp.setattr("py_mmd_tools.mmd_operations.shutil.move",
+                   lambda *a, **k: None)
+        mp.setattr("py_mmd_tools.mmd_operations.requests.get",
+                   lambda *a, **k: MockResponse())
+        mp.setattr("py_mmd_tools.mmd_operations.requests.post",
+                   lambda *a, **k: MockResponse())
+        not_updated, updated = main(parsed)
+        assert len(not_updated) == 0
+        assert len(updated) == 1
+        assert os.path.isfile(updated[0])
+        lines = mmd_readlines(updated[0])
         for line in lines:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
@@ -48,7 +59,8 @@ def test_main(dataDir, monkeypatch):
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",
                    lambda *a, **k: False)
         with pytest.raises(ValueError):
-            updated, mmd_new = main(parsed)
+            not_updated, updated = main(parsed)
+        assert len(updated) == 1
 
         map = {
             mmd_repository_path: True,
@@ -59,7 +71,8 @@ def mock_isdir(pp):
 
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", mock_isdir)
         with pytest.raises(ValueError):
-            updated, mmd_new = main(parsed)
+            not_updated, updated = main(parsed)
 
+        assert len(updated) == 1
         # Remove new MMD file
-        os.remove(mmd_new)
+        os.remove(updated[0])

From dd7232fba2d11b03b2ba68bbf09143885f58dfbf Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 18:26:59 +0200
Subject: [PATCH 06/32] #337: executable script

---
 py_mmd_tools/script/move_data.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 py_mmd_tools/script/move_data.py

diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
old mode 100644
new mode 100755

From 6e8b03578a87c61dae2237d220342abbfb8ffcf4 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 18:29:15 +0200
Subject: [PATCH 07/32] #337: add dependency

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 8ab8b446..ba5ed403 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,6 +38,7 @@ dependencies = [
 	"shapely",
 	"wget",
 	"xmltodict",
+        "datetime_glob",
 ]
 name = "py-mmd-tools"
 description = "This  is a tools for generating MMD files from netCDF-CF files with ACDD attributes, for documenting netCDF-CF files from MMD information."

From 6998b5eaf38cbf4a5b7b2da7219a5491d6a8afcb Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 18:58:34 +0200
Subject: [PATCH 08/32] #337: fix flake errors

---
 py_mmd_tools/mmd_operations.py   | 43 ++++++++++++++++----------------
 py_mmd_tools/script/move_data.py | 14 +----------
 tests/test_mmd_operations.py     |  5 ++--
 tests/test_move_data_script.py   |  1 +
 4 files changed, 27 insertions(+), 36 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 5200fe3c..053eb807 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -21,12 +21,13 @@
 
 def add_metadata_update_info(f, note, type="Minor modification"):
     """ Add update information """
-    f.write("    <mmd:update>\n"
-            "      <mmd:datetime>%s</mmd:datetime>\n"
-            "      <mmd:type>%s</mmd:type>\n"
-            "      <mmd:note>%s</mmd:note>\n"
-            "    </mmd:update>\n" % (datetime.datetime.utcnow().replace(
-        tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), type, note))
+    f.write(
+        "    <mmd:update>\n"
+        "      <mmd:datetime>%s</mmd:datetime>\n"
+        "      <mmd:type>%s</mmd:type>\n"
+        "      <mmd:note>%s</mmd:note>\n"
+        "    </mmd:update>\n" % (datetime.datetime.utcnow().replace(
+                                 tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), type, note))
 
 
 def get_local_mmd_git_path(nc_file, mmd_repository_path):
@@ -103,7 +104,6 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
         }
     }
 
-
     if os.path.isfile(existing_pathname_pattern_or_exact):
         existing = [existing_pathname_pattern_or_exact]
         existing_pathname_pattern = None
@@ -129,10 +129,6 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
             data = fn.read()
 
         res = requests.post(url=f"https://{urls[env]['dmci']}/v1/validate", data=data)
-        dmci_valid = False
-        if res.status_code == 200:
-            dmci_valid = True
-
 
         # Update with dmci update
         dmci_updated = False
@@ -146,20 +142,25 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
 
         # If update was ok - move netcdf file
         nc_moved = False
-        if dmci_updated:
+        if dmci_updated and not dry_run:
             shutil.move(file, nfl)
             nc_moved = True
+        elif dmci_updated and dry_run:
+            nc_moved = True
 
         # Check by searching CSW and checking data access urls
-        res = requests.get(url=f"https://{urls[env]['csw']}/csw",
-                           params={
-                               "service": "CSW",
-                               "version": "2.0.2",
-                               "request": "GetRepositoryItem",
-                               "id": f"no.met.{urls[env]['id_namespace']}"
-                                     f"{os.path.basename(file).split('.')[0]}"})
         ds_found_and_accessible = False
-        if res.status_code == 200:
+        if not dry_run:
+            res = requests.get(url=f"https://{urls[env]['csw']}/csw",
+                               params={
+                                   "service": "CSW",
+                                   "version": "2.0.2",
+                                   "request": "GetRepositoryItem",
+                                   "id": f"no.met.{urls[env]['id_namespace']}"
+                                         f"{os.path.basename(file).split('.')[0]}"})
+            if res.status_code == 200:
+                ds_found_and_accessible = True
+        else:
             ds_found_and_accessible = True
 
         if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]):
@@ -176,7 +177,7 @@ def new_file_location(file, new_base_loc, existing_pathname_pattern=None):
     new_base_loc. This is to allow usage flexibility of the move_data
     function.
     """
-    if existing_pathname_pattern == None:
+    if existing_pathname_pattern is None:
         file_path = os.path.join(new_base_loc, os.path.basename(file))
     else:
         file_path = os.path.join(new_base_loc, file.removeprefix(re.split(r"[^\w/-]",
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index d6e6663b..d3dc0498 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -15,21 +15,9 @@
     move_data [-h] -i INPUT -n OUTPUT_DIR
 """
 import os
-import re
-import glob
-import uuid
-import netCDF4
-import pathlib
 import argparse
-import tempfile
-import datetime_glob
 
 from py_mmd_tools.mmd_operations import move_data
-from py_mmd_tools.mmd_operations import mmd_readlines
-from py_mmd_tools.mmd_operations import new_file_location
-from py_mmd_tools.mmd_operations import get_local_mmd_git_path
-from py_mmd_tools.mmd_operations import add_metadata_update_info
-from py_mmd_tools.mmd_operations import mmd_change_file_location
 
 
 def create_parser():
@@ -53,7 +41,7 @@ def create_parser():
         '--dmci-update', action='store_true',
         help='Directly update the online catalog with the changed MMD files.'
     )
-        
+
     return parser
 
 
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 43e21a4b..1c21216d 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -28,6 +28,7 @@ def test_get_local_mmd_git_path(dataDir):
     assert fn == "/some/folder/mmd-xml-production/arch_4/arch_3/arch_9/" \
                  "b7cb7934-77ca-4439-812e-f560df3fe7eb.xml"
 
+
 @pytest.mark.script
 def test_mmd_change_file_location(dataDir):
     """Test that an MMD file is created with new file_location, and
@@ -38,7 +39,7 @@ def test_mmd_change_file_location(dataDir):
     new_file_location = "/some/where/else/2024/06/19"
 
     new_mmd, changed = mmd_change_file_location(mmd, new_file_location)
-    assert changed == True
+    assert changed is True
     assert os.path.isfile(new_mmd)
     lines = mmd_readlines(new_mmd)
     for line in lines:
@@ -46,7 +47,7 @@ def test_mmd_change_file_location(dataDir):
             assert "/some/where/else/2024/06/19" in line
 
     mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False)
-    assert changed == True
+    assert changed is True
     assert os.path.isfile(mmd)
     lines = mmd_readlines(mmd)
     for line in lines:
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index 5aab9402..a3413350 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -66,6 +66,7 @@ class MockResponse:
             mmd_repository_path: True,
             new_file_location_base: False
         }
+
         def mock_isdir(pp):
             return map[pp]
 

From de70609fc0d320e54e3095515c2c068f98a2c9fc Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 18:32:22 +0000
Subject: [PATCH 09/32] #337: updates after real-life test

---
 py_mmd_tools/mmd_operations.py   | 15 ++++++++-------
 py_mmd_tools/script/move_data.py | 12 +++++++-----
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 053eb807..e2c6eb40 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -172,16 +172,17 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
 
 
 def new_file_location(file, new_base_loc, existing_pathname_pattern=None):
-    """Return the new file location. If existing_pathname_pattern is
-    None, the returned path will equal the provided parameter
-    new_base_loc. This is to allow usage flexibility of the move_data
-    function.
+    """Return the name of the new folder where the netcdf file will be
+    stored. If existing_pathname_pattern is None, the returned path
+    will equal the provided parameter new_base_loc. This is to allow
+    usage flexibility of the move_data function.
     """
     if existing_pathname_pattern is None:
         file_path = os.path.join(new_base_loc, os.path.basename(file))
     else:
         file_path = os.path.join(new_base_loc, file.removeprefix(re.split(r"[^\w/-]",
                                                                  existing_pathname_pattern)[0]))
-    if not os.path.isfile(file_path):
-        raise ValueError(f"File does not exist: {file_path}")
-    return os.path.dirname(os.path.abspath(file_path))
+    new_folder = os.path.dirname(os.path.abspath(file_path))
+    if not os.path.isdir(new_folder):
+        raise ValueError(f"Folder does not exist: {file_path}")
+    return new_folder
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index d3dc0498..b585ab74 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -30,7 +30,7 @@ def create_parser():
         help="Local folder containing all MMD files.")
     parser.add_argument(
         "new_file_location_base", type=str,
-        help="Base or exact path to the new file location.")
+        help="Base or exact path to the folder to which the new file will be moved.")
     parser.add_argument(
         "existing_pathname_pattern", type=str,
         help="Pathname pattern to existing file location(s). Allows "
@@ -54,10 +54,12 @@ def main(args=None):
     if not os.path.isdir(args.new_file_location_base):
         raise ValueError(f"Invalid input: {args.new_file_location_base}")
 
-    return move_data(args.mmd_repository_path,
-                     args.new_file_location_base,
-                     args.existing_pathname_pattern,
-                     dry_run=not args.dmci_update)
+    not_updated, updated =  move_data(args.mmd_repository_path,
+                                      args.new_file_location_base,
+                                      args.existing_pathname_pattern,
+                                      dry_run=not args.dmci_update)
+    print(f"Updated: {len(updated)}")
+    print(f"Not updated: {len(not_updated)}")
 
 
 def _main():  # pragma: no cover

From 21088af75ddd3cade799809155ff1050a20480ab Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 20:07:45 +0000
Subject: [PATCH 10/32] #337: minor bug fix

---
 py_mmd_tools/mmd_operations.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index e2c6eb40..aef34edf 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -112,17 +112,19 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
                     datetime_glob.walk(pattern=existing_pathname_pattern_or_exact)]
         existing_pathname_pattern = existing_pathname_pattern_or_exact
 
+    copy_mmd = False
     if dry_run:
         # Not copying the file will make it easy to check changes
         # with git diff
         existing_pathname_pattern = None
+        copy_mmd = True
 
     updated = []
     not_updated = []
     for file in existing:
         nfl = new_file_location(file, new_file_location_base, existing_pathname_pattern)
         mmd_orig = get_local_mmd_git_path(file, mmd_repository_path)
-        mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl)
+        mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl, copy=copy_mmd)
 
         # Get MMD content as binary data
         with open(mmd_new, "rb") as fn:

From 666e9eaa6b1923fa369dc5166b161e9734a7ba77 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 29 Jul 2024 20:11:30 +0000
Subject: [PATCH 11/32] #337: another minor bug fix

---
 py_mmd_tools/mmd_operations.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index aef34edf..1e2a11de 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -112,12 +112,12 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
                     datetime_glob.walk(pattern=existing_pathname_pattern_or_exact)]
         existing_pathname_pattern = existing_pathname_pattern_or_exact
 
-    copy_mmd = False
+    copy_mmd = True
     if dry_run:
         # Not copying the file will make it easy to check changes
         # with git diff
         existing_pathname_pattern = None
-        copy_mmd = True
+        copy_mmd = False
 
     updated = []
     not_updated = []

From 376011cab0f027aa68aa4952cca6f4a3dcf5a946 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Tue, 30 Jul 2024 12:01:45 +0000
Subject: [PATCH 12/32] #337: changes after testing on actual data

---
 py_mmd_tools/mmd_operations.py   | 116 ++++++++++++++++++++++---------
 py_mmd_tools/script/move_data.py |   4 +-
 2 files changed, 87 insertions(+), 33 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 1e2a11de..e934f499 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -30,6 +30,27 @@ def add_metadata_update_info(f, note, type="Minor modification"):
                                  tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), type, note))
 
 
+def check_csw_catalog(ds_id, nc_file, urls, env, emsg):
+    """Search for the dataset with id 'ds_id' in the CSW metadata
+    catalog.
+    """
+    ds_found_and_accessible = False
+    res = requests.get(url=f"https://{urls[env]['csw']}/csw",
+                        params={
+                            "service": "CSW",
+                            "version": "2.0.2",
+                            "request": "GetRepositoryItem",
+                            "id": ds_id})
+    # TODO: check the data_access urls
+    if res.status_code == 200:
+        ds_found_and_accessible = True
+    else:
+        emsg += (f"Could not find dataset in CSW catalog: "
+                 f"{os.path.basename(nc_file).split('.')[0]}")
+
+    return ds_found_and_accessible, emsg
+
+
 def get_local_mmd_git_path(nc_file, mmd_repository_path):
     """Return the path to the original MMD file.
     """
@@ -43,10 +64,12 @@ def get_local_mmd_git_path(nc_file, mmd_repository_path):
 
 
 def mmd_change_file_location(mmd, new_file_location, copy=True):
-    """Copy original MMD file, and make changes. Return the filename
-    of the updated file, and a status flag indicating if it has been
-    changed or not.
+    """Copy original MMD file, and change the file_location field.
+    Return the filename of the updated MMD file, and a status flag
+    indicating if it has been changed or not.
     """
+    if not os.path.isfile(mmd):
+        return None, False
     if copy:
         tmp_path = tempfile.gettempdir()
         shutil.copy2(mmd, tmp_path)
@@ -77,6 +100,20 @@ def mmd_readlines(filename):
     return lines
 
 
+def move_data_file(nc_file, nfl, emsg=""):
+    """Move data file from nc_file to nfl.
+    """
+    nc_moved = False
+    try:
+        shutil.move(nc_file, nfl)
+    except Exception as e:
+        nc_moved = False
+        emsg = f"Could not move file from {nc_file} to {nfl}.\nError message: {str(e)}\n"
+    else:
+        nc_moved = True
+    return nc_moved, emsg
+
+
 def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pattern_or_exact,
               dry_run=True, env="prod"):
     """Update MMD and move data file.
@@ -108,82 +145,97 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
         existing = [existing_pathname_pattern_or_exact]
         existing_pathname_pattern = None
     else:
-        existing = [file for match, file in
+        existing = [str(nc_file) for match, nc_file in
                     datetime_glob.walk(pattern=existing_pathname_pattern_or_exact)]
         existing_pathname_pattern = existing_pathname_pattern_or_exact
 
     copy_mmd = True
     if dry_run:
-        # Not copying the file will make it easy to check changes
+        # Not copying the MMD file will make it easy to check changes
         # with git diff
         existing_pathname_pattern = None
         copy_mmd = False
 
     updated = []
-    not_updated = []
-    for file in existing:
-        nfl = new_file_location(file, new_file_location_base, existing_pathname_pattern)
-        mmd_orig = get_local_mmd_git_path(file, mmd_repository_path)
+    not_updated = {}
+    for nc_file in existing:
+        # Error message
+        emsg = ""
+        nfl = new_file_location(nc_file, new_file_location_base, existing_pathname_pattern)
+        try:
+            mmd_orig = get_local_mmd_git_path(nc_file, mmd_repository_path)
+        except Exception as e:
+            not_updated[nc_file] = f"Could not get MMD path of {nc_file}.\nError: {str(e)}"
+            continue
+
+        # Check permissions before doing anything
+        remove_file_allowed = os.access(nc_file, os.W_OK)
+        write_file_allowed = os.access(nfl, os.W_OK)
+        if not remove_file_allowed or not write_file_allowed:
+            if not remove_file_allowed:
+                not_updated[mmd_orig] = f"Missing permission to delete {nc_file}"
+            if not write_file_allowed:
+                not_updated[mmd_orig] = f"Missing permission to write {nfl}"
+            if not remove_file_allowed and not write_file_allowed:
+                not_updated[mmd_orig] = (f"Missing permission to delete {nc_file} "
+                                         f"nor to write {nfl}")
+            continue
+
         mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl, copy=copy_mmd)
+        if not mmd_updated:
+            emsg = f"Could not update MMD file for {nc_file}"
+            not_updated[mmd_orig] = emsg
+            continue
 
         # Get MMD content as binary data
         with open(mmd_new, "rb") as fn:
             data = fn.read()
-
         res = requests.post(url=f"https://{urls[env]['dmci']}/v1/validate", data=data)
 
         # Update with dmci update
         dmci_updated = False
-        if not dry_run:
+        if res.status_code == 200 and not dry_run:
             # be careful with this...
             res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data)
         if res.status_code == 200:
             # This intentionally becomes True in case of dry-run and
             # a valid xml
             dmci_updated = True
+        else:
+            emsg = "Could not push updated MMD file to the DMCI API."
+            not_updated[mmd_orig] = emsg
+            continue
 
-        # If update was ok - move netcdf file
-        nc_moved = False
         if dmci_updated and not dry_run:
-            shutil.move(file, nfl)
-            nc_moved = True
+            nc_moved, emsg = move_data_file(nc_file, nfl)
         elif dmci_updated and dry_run:
             nc_moved = True
 
-        # Check by searching CSW and checking data access urls
-        ds_found_and_accessible = False
+        ds_id = f"no.met.{urls[env]['id_namespace']}:{os.path.basename(mmd_orig).split('.')[0]}"
         if not dry_run:
-            res = requests.get(url=f"https://{urls[env]['csw']}/csw",
-                               params={
-                                   "service": "CSW",
-                                   "version": "2.0.2",
-                                   "request": "GetRepositoryItem",
-                                   "id": f"no.met.{urls[env]['id_namespace']}"
-                                         f"{os.path.basename(file).split('.')[0]}"})
-            if res.status_code == 200:
-                ds_found_and_accessible = True
+            ds_found_and_accessible, emsg = check_csw_catalog(ds_id, nc_file, urls, env, emsg)
         else:
             ds_found_and_accessible = True
 
         if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]):
-            updated.append(mmd_new)
+            updated.append(mmd_orig)
         else:
-            not_updated.append(mmd_new)
+            not_updated[mmd_orig] = emsg
 
     return not_updated, updated
 
 
-def new_file_location(file, new_base_loc, existing_pathname_pattern=None):
+def new_file_location(nc_file, new_base_loc, existing_pathname_pattern=None):
     """Return the name of the new folder where the netcdf file will be
     stored. If existing_pathname_pattern is None, the returned path
     will equal the provided parameter new_base_loc. This is to allow
     usage flexibility of the move_data function.
     """
     if existing_pathname_pattern is None:
-        file_path = os.path.join(new_base_loc, os.path.basename(file))
+        file_path = os.path.join(new_base_loc, os.path.basename(nc_file))
     else:
-        file_path = os.path.join(new_base_loc, file.removeprefix(re.split(r"[^\w/-]",
-                                                                 existing_pathname_pattern)[0]))
+        prefix = re.split(r"[^\w/-]", existing_pathname_pattern)[0]
+        file_path = os.path.join(new_base_loc, nc_file[len(prefix):])
     new_folder = os.path.dirname(os.path.abspath(file_path))
     if not os.path.isdir(new_folder):
         raise ValueError(f"Folder does not exist: {file_path}")
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index b585ab74..45a76ae5 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -59,7 +59,9 @@ def main(args=None):
                                       args.existing_pathname_pattern,
                                       dry_run=not args.dmci_update)
     print(f"Updated: {len(updated)}")
-    print(f"Not updated: {len(not_updated)}")
+    print(f"Not updated: {len(not_updated)}\n")
+    for key, val in not_updated.items():
+        print(f"{key}: {val}")
 
 
 def _main():  # pragma: no cover

From fa87f4fd3d8bda7713109562cb844db221d4484e Mon Sep 17 00:00:00 2001
From: Morten Wergeland Hansen <mortenwh@met.no>
Date: Tue, 30 Jul 2024 14:40:35 +0200
Subject: [PATCH 13/32] #337: still updating tests

---
 tests/data/reference_nc.xml  | 17 ++++++++++++++++-
 tests/test_mmd_operations.py |  8 ++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/tests/data/reference_nc.xml b/tests/data/reference_nc.xml
index 9a4e79a1..27c768f9 100644
--- a/tests/data/reference_nc.xml
+++ b/tests/data/reference_nc.xml
@@ -12,6 +12,21 @@
       <mmd:datetime>2020-11-27T14:05:56Z</mmd:datetime>
       <mmd:type>Created</mmd:type>
     </mmd:update>
+    <mmd:update>
+      <mmd:datetime>2024-07-30T12:36:53Z</mmd:datetime>
+      <mmd:type>Minor modification</mmd:type>
+      <mmd:note>New file location in storage information.</mmd:note>
+    </mmd:update>
+    <mmd:update>
+      <mmd:datetime>2024-07-30T12:39:39Z</mmd:datetime>
+      <mmd:type>Minor modification</mmd:type>
+      <mmd:note>New file location in storage information.</mmd:note>
+    </mmd:update>
+    <mmd:update>
+      <mmd:datetime>2024-07-30T12:39:57Z</mmd:datetime>
+      <mmd:type>Minor modification</mmd:type>
+      <mmd:note>New file location in storage information.</mmd:note>
+    </mmd:update>
   </mmd:last_metadata_update>
   <mmd:temporal_extent>
     <mmd:start_date>2020-11-27T13:40:02.019817Z</mmd:start_date>
@@ -142,7 +157,7 @@
   </mmd:data_access>
   <mmd:storage_information>
     <mmd:file_name>reference_nc.nc</mmd:file_name>
-    <mmd:file_location>/home/mortenwh/dev/py-mmd-tools/tests/data</mmd:file_location>
+    <mmd:file_location>/some/where/new</mmd:file_location>
     <mmd:file_format>NetCDF-CF</mmd:file_format>
     <mmd:file_size unit="MB">0.03</mmd:file_size>
     <mmd:checksum type="md5sum">aa188b45ece9bdbbc9b470106a6b13f8</mmd:checksum>
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 1c21216d..c41e544f 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -102,6 +102,8 @@ class MockResponse:
                    lambda *a, **k: MockResponse())
         mp.setattr("py_mmd_tools.mmd_operations.requests.post",
                    lambda *a, **k: MockResponse())
+        mp.setattr("py_mmd_tools.mmd_operations.os.access",
+                   lambda *a, **k: True)
         not_updated, updated = move_data(mmd_repository_path, new_file_location_base, nc_file)
         assert len(not_updated) == 0
         assert len(updated) == 1
@@ -110,8 +112,6 @@ class MockResponse:
         for line in lines:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
-        # Remove new MMD file
-        os.remove(updated[0])
 
         def mock_walk(*a, **k):
             yield (1, nc_file)
@@ -130,8 +130,8 @@ def mock_walk(*a, **k):
         for line in lines:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
-        # Remove new MMD file
-        os.remove(updated[0])
+        # TODO: Remove new MMD file - manual update needed for now..
+        # os.remove() - get filename through a function
 
         mp.setattr("py_mmd_tools.mmd_operations.mmd_change_file_location",
                    lambda *a, **k: (nc_file, False))

From 57a93ad376ce2b7a1a703c2c326bc3bc3964e117 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 31 Jul 2024 14:07:36 +0200
Subject: [PATCH 14/32] #337: Full test coverage

---
 py_mmd_tools/mmd_operations.py |   7 +-
 tests/test_mmd_operations.py   | 140 ++++++++++++++++++++++++++++++---
 tests/test_move_data_script.py |  19 ++---
 3 files changed, 139 insertions(+), 27 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index e934f499..efb46423 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -30,7 +30,7 @@ def add_metadata_update_info(f, note, type="Minor modification"):
                                  tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), type, note))
 
 
-def check_csw_catalog(ds_id, nc_file, urls, env, emsg):
+def check_csw_catalog(ds_id, nc_file, urls, env, emsg=""):
     """Search for the dataset with id 'ds_id' in the CSW metadata
     catalog.
     """
@@ -45,8 +45,7 @@ def check_csw_catalog(ds_id, nc_file, urls, env, emsg):
     if res.status_code == 200:
         ds_found_and_accessible = True
     else:
-        emsg += (f"Could not find dataset in CSW catalog: "
-                 f"{os.path.basename(nc_file).split('.')[0]}")
+        emsg += f"Could not find dataset in CSW catalog: {nc_file} (id: {ds_id})"
 
     return ds_found_and_accessible, emsg
 
@@ -213,7 +212,7 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
 
         ds_id = f"no.met.{urls[env]['id_namespace']}:{os.path.basename(mmd_orig).split('.')[0]}"
         if not dry_run:
-            ds_found_and_accessible, emsg = check_csw_catalog(ds_id, nc_file, urls, env, emsg)
+            ds_found_and_accessible, emsg = check_csw_catalog(ds_id, nc_file, urls, env, emsg=emsg)
         else:
             ds_found_and_accessible = True
 
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index c41e544f..60c67568 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -10,15 +10,20 @@
 import os
 import pytest
 import shutil
+import subprocess
 
+from unittest.mock import Mock
+
+from py_mmd_tools.mmd_operations import check_csw_catalog
 from py_mmd_tools.mmd_operations import move_data
+from py_mmd_tools.mmd_operations import move_data_file
 from py_mmd_tools.mmd_operations import mmd_readlines
 from py_mmd_tools.mmd_operations import new_file_location
 from py_mmd_tools.mmd_operations import mmd_change_file_location
 from py_mmd_tools.mmd_operations import get_local_mmd_git_path
 
 
-@pytest.mark.script
+@pytest.mark.py_mmd_tools
 def test_get_local_mmd_git_path(dataDir):
     """Test that the mmd git path is returned correctly with folders
     composed from its uuid.
@@ -29,7 +34,7 @@ def test_get_local_mmd_git_path(dataDir):
                  "b7cb7934-77ca-4439-812e-f560df3fe7eb.xml"
 
 
-@pytest.mark.script
+@pytest.mark.py_mmd_tools
 def test_mmd_change_file_location(dataDir):
     """Test that an MMD file is created with new file_location, and
     new metadata update.
@@ -58,8 +63,13 @@ def test_mmd_change_file_location(dataDir):
     os.remove(mmd)
     os.remove(new_mmd)
 
+    # Test that it fails when the mmd does not exist
+    mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False)
+    assert mmd is None
+    assert changed is False
+
 
-@pytest.mark.script
+@pytest.mark.py_mmd_tools
 def test_mmd_readlines(dataDir):
     """Test that the lines in the MMD file are returned as a list.
     """
@@ -73,7 +83,7 @@ def test_mmd_readlines(dataDir):
         lines = mmd_readlines(mmd)
 
 
-@pytest.mark.script
+@pytest.mark.py_mmd_tools
 def test_move_data(dataDir, monkeypatch):
     """Test the move_data function.
     """
@@ -81,6 +91,9 @@ def test_move_data(dataDir, monkeypatch):
     new_file_location_base = "/some/where/new"
     nc_file = os.path.join(dataDir, "reference_nc.nc")
 
+    def mock_walk(*a, **k):
+        yield (1, nc_file)
+
     class MockResponse:
 
         status_code = 200
@@ -113,9 +126,6 @@ class MockResponse:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
 
-        def mock_walk(*a, **k):
-            yield (1, nc_file)
-
         mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
                    lambda *a, **k: mock_walk(*a, **k))
         # This pattern is not really used but it should look something
@@ -138,8 +148,77 @@ def mock_walk(*a, **k):
         not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
         assert len(not_updated) == 1
 
+        def raise_(ex):
+            raise ex
+
+        mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
+                   lambda *a, **k: raise_(Exception("No path")))
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+        assert "Could not get MMD path" in not_updated[list(not_updated.keys())[0]]
+
+    # Test os.access, remove_file_allowed is False
+    mock_access = Mock()
+    mock_access.side_effect = [False, True]
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
+                   lambda *a, **k: mock_walk(*a, **k))
+        mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
+                   lambda *a, **k: new_file_location_base)
+        mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access)
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+
+    # Test os.access, remove_file_allowed and write_file_allowed are
+    # False
+    mock_access = Mock()
+    mock_access.side_effect = [False, False]
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
+                   lambda *a, **k: mock_walk(*a, **k))
+        mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
+                   lambda *a, **k: new_file_location_base)
+        mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access)
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+
+    # Test second call to requests.post fails (dmci update)
+    class MockResponseFail:
 
-@pytest.mark.script
+        status_code = 400
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
+                   lambda *a, **k: mock_walk(*a, **k))
+        mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
+                   lambda *a, **k: new_file_location_base)
+        mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
+                   lambda *a, **k: os.path.join(dataDir, "reference_nc.xml"))
+        mp.setattr("py_mmd_tools.mmd_operations.shutil.move",
+                   lambda *a, **k: None)
+        mp.setattr("py_mmd_tools.mmd_operations.os.access",
+                   lambda *a, **k: True)
+        mp.setattr("py_mmd_tools.mmd_operations.requests.get",
+                   lambda *a, **k: MockResponse())
+        mock_post = Mock()
+        mock_post.side_effect = [MockResponse(), MockResponseFail()]
+        mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern,
+                                         dry_run=False)
+        assert "Could not push updated" in not_updated[os.path.join(dataDir, "reference_nc.xml")]
+
+        # Check that updated is False if check_csw_catalog fails
+        mock_post = Mock()
+        mock_post.side_effect = [MockResponse(), MockResponse()]
+        mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
+        mp.setattr("py_mmd_tools.mmd_operations.check_csw_catalog",
+                   lambda *a, **k: (False, "Fail"))
+        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern,
+                                         dry_run=False)
+        assert not_updated[os.path.join(dataDir, "reference_nc.xml")] == "Fail"
+
+    subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+
+
+@pytest.mark.py_mmd_tools
 def test_new_file_location(monkeypatch):
     """Test that the returned new location paths are correct.
     """
@@ -152,8 +231,51 @@ def test_new_file_location(monkeypatch):
         new_file_location(file, new_base, existing_pathname_pattern)
 
     with monkeypatch.context() as mp:
-        mp.setattr("py_mmd_tools.mmd_operations.os.path.isfile", lambda *a, **k: True)
+        mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", lambda *a, **k: True)
         assert new_file_location(file, new_base, existing_pathname_pattern) == \
             "/some/where/else/2024/06/19"
         new_base = "/some/where/else/2024/06/19"
         assert new_file_location(file, new_base) == "/some/where/else/2024/06/19"
+
+
+@pytest.mark.py_mmd_tools
+def test_check_csw_catalog(monkeypatch):
+    """Test check_csw_catalog
+    """
+    ds_id = "no.met:123"
+    nc_file = "/some/file.nc"
+    urls = {
+        "prod": {
+            "csw": "data.csw.some-place.no",
+        }
+    }
+    env = "prod"
+
+    class MockResponse:
+
+        status_code = 400
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.requests.get",
+                   lambda *a, **k: MockResponse())
+        found, msg = check_csw_catalog(ds_id, nc_file, urls, env)
+        assert found is False
+        assert msg == "Could not find dataset in CSW catalog: /some/file.nc (id: no.met:123)"
+
+
+@pytest.mark.py_mmd_tools
+def test_move_data_file(monkeypatch):
+    """Test move_data_file function can't move (working move is
+    tested in test_move_data).
+    """
+    nc_file = "/some/file.nc"
+    nfl = "/some/new/location"
+
+    def raise_(ex):
+        raise ex
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.shutil.move",
+                   lambda *a, **k: raise_(Exception("Permission error...")))
+        moved, msg = move_data_file(nc_file, nfl)
+        assert moved is False
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index a3413350..d8222708 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -9,6 +9,7 @@
 """
 import os
 import pytest
+import subprocess
 
 from py_mmd_tools.mmd_operations import mmd_readlines
 
@@ -47,20 +48,12 @@ class MockResponse:
                    lambda *a, **k: MockResponse())
         mp.setattr("py_mmd_tools.mmd_operations.requests.post",
                    lambda *a, **k: MockResponse())
-        not_updated, updated = main(parsed)
-        assert len(not_updated) == 0
-        assert len(updated) == 1
-        assert os.path.isfile(updated[0])
-        lines = mmd_readlines(updated[0])
-        for line in lines:
-            if "<mmd:file_location>" in line:
-                assert "/some/where/new" in line
+        main(parsed)
 
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",
                    lambda *a, **k: False)
         with pytest.raises(ValueError):
-            not_updated, updated = main(parsed)
-        assert len(updated) == 1
+            main(parsed)
 
         map = {
             mmd_repository_path: True,
@@ -72,8 +65,6 @@ def mock_isdir(pp):
 
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", mock_isdir)
         with pytest.raises(ValueError):
-            not_updated, updated = main(parsed)
+            main(parsed)
 
-        assert len(updated) == 1
-        # Remove new MMD file
-        os.remove(updated[0])
+    subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])

From 5cb79c96f4e2167d8e1849677614ce93af6033b1 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Thu, 1 Aug 2024 11:19:08 +0200
Subject: [PATCH 15/32] #337: fix flake errors

---
 py_mmd_tools/mmd_operations.py   | 10 +++++-----
 py_mmd_tools/script/move_data.py |  8 ++++----
 tests/test_move_data_script.py   |  2 --
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index efb46423..218d6b19 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -36,11 +36,11 @@ def check_csw_catalog(ds_id, nc_file, urls, env, emsg=""):
     """
     ds_found_and_accessible = False
     res = requests.get(url=f"https://{urls[env]['csw']}/csw",
-                        params={
-                            "service": "CSW",
-                            "version": "2.0.2",
-                            "request": "GetRepositoryItem",
-                            "id": ds_id})
+                       params={
+                           "service": "CSW",
+                           "version": "2.0.2",
+                           "request": "GetRepositoryItem",
+                           "id": ds_id})
     # TODO: check the data_access urls
     if res.status_code == 200:
         ds_found_and_accessible = True
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index 45a76ae5..9a0e83ca 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -54,10 +54,10 @@ def main(args=None):
     if not os.path.isdir(args.new_file_location_base):
         raise ValueError(f"Invalid input: {args.new_file_location_base}")
 
-    not_updated, updated =  move_data(args.mmd_repository_path,
-                                      args.new_file_location_base,
-                                      args.existing_pathname_pattern,
-                                      dry_run=not args.dmci_update)
+    not_updated, updated = move_data(args.mmd_repository_path,
+                                     args.new_file_location_base,
+                                     args.existing_pathname_pattern,
+                                     dry_run=not args.dmci_update)
     print(f"Updated: {len(updated)}")
     print(f"Not updated: {len(not_updated)}\n")
     for key, val in not_updated.items():
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index d8222708..06e6694e 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -11,8 +11,6 @@
 import pytest
 import subprocess
 
-from py_mmd_tools.mmd_operations import mmd_readlines
-
 from py_mmd_tools.script.move_data import main
 from py_mmd_tools.script.move_data import create_parser
 

From e0097cb933f5e20c805416e439a137c6905b6882 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Thu, 1 Aug 2024 09:41:58 +0000
Subject: [PATCH 16/32] Add description to README

---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index 10c87d48..50fab451 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,15 @@ Python tools for MMD. The package contains tools for generating MMD files from n
 
 generates an output MMD file called `reference_nc.xml`.
 
+In addition, the `mmd_operations` module currently contains a tool to move data
+files and accordingly update MMD files registered in online catalogs. This
+module can be extended with other necessary data management tools. Moving
+can be done with the `move_data` script, e.g.:
+
+```
+move_data /path/to/files-from-git/mmd-xml-<env> /path/to/new/storage "/path/to/data/files/*.nc" --dmci-update
+```
+
 # Installation
 
 To avoid problems with conflicting versions, we recommend using the [Conda](

From 2e3d411e069b7e523fddc44f732e005f3fcbea41 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 30 Sep 2024 12:42:39 +0200
Subject: [PATCH 17/32] #337: install new script

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index ba5ed403..c0115d6f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,7 @@ nc2mmd = "py_mmd_tools.script.nc2mmd:_main"
 check_nc = "py_mmd_tools.script.check_nc:_main"
 yaml2adoc = "py_mmd_tools.script.yaml2adoc:_main"
 ncheader2json = "py_mmd_tools.script.ncheader2json:_main"
+move_data = "py_mmd_tools.script.move_data:_main"
 
 [project.urls]
 source = "https://github.com/metno/py-mmd-tools"

From 50b4edb53f72f555cdf9f175f862abf483e0948f Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 30 Sep 2024 13:48:05 +0200
Subject: [PATCH 18/32] #337: properly close nc files

---
 tests/test_nc_to_mmd.py | 634 ++++++++++++++++++++--------------------
 1 file changed, 325 insertions(+), 309 deletions(-)

diff --git a/tests/test_nc_to_mmd.py b/tests/test_nc_to_mmd.py
index 7aab7819..da5590c1 100644
--- a/tests/test_nc_to_mmd.py
+++ b/tests/test_nc_to_mmd.py
@@ -120,14 +120,15 @@ def test_separate_repeated(dataDir):
     """ Test function Nc_to_mmd.separate_repeated
     """
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.platform = ['&quot;Basis&quot;: &quot;Space-based Platforms&quot;',
-                     ' &quot;Category&quot;: &quot;Earth Observation Satellites&quot;',
-                     ' &quot;Sub_Category&quot;: &quot;Sentinel-1&quot;',
-                     ' &quot;Short_Name&quot;: &quot;Sentinel-1A&quot;',
-                     ' &quot;Long_Name&quot;: &quot;Sentinel-1A&quot;']
-    with pytest.raises(AttributeError) as ee:
-        md.separate_repeated(True, getattr(ncin, "platform"))
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.platform = ['&quot;Basis&quot;: &quot;Space-based Platforms&quot;',
+                        ' &quot;Category&quot;: &quot;Earth Observation Satellites&quot;',
+                        ' &quot;Sub_Category&quot;: &quot;Sentinel-1&quot;',
+                        ' &quot;Short_Name&quot;: &quot;Sentinel-1A&quot;',
+                        ' &quot;Long_Name&quot;: &quot;Sentinel-1A&quot;']
+        with pytest.raises(AttributeError) as ee:
+            md.separate_repeated(True, getattr(ncin, "platform"))
+
     assert str(ee.value) == "'list' object has no attribute 'split'"
 
 
@@ -137,20 +138,21 @@ def testNc_to_mmd_get_geographic_extent_polygon(dataDir):
     geospatial_bounds_crs is missing.
     """
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.geospatial_bounds = ("POLYGON ((59.01 1.23, 59.06 1.66, 59.10 2.09, 59.15 2.53, "
-                              "59.19 2.96, 59.24 3.40, 59.28 3.84, 59.32 4.28, 59.35 4.72, "
-                              "59.39 5.16, 59.43 5.61, 59.43 5.61, 59.57 5.57, 59.72 5.52, "
-                              "59.87 5.48, 60.01 5.43, 60.16 5.39, 60.31 5.34, 60.46 5.29, "
-                              "60.60 5.25, 60.75 5.20, 60.92 5.15, 60.92 5.15, 60.88 4.67, "
-                              "60.84 4.21, 60.80 3.75, 60.76 3.29, 60.72 2.84, 60.68 2.38, "
-                              "60.63 1.92, 60.59 1.47, 60.54 1.02, 60.49 0.56, 60.49 0.56, "
-                              "60.33 0.64, 60.18 0.71, 60.03 0.77, 59.89 0.84, 59.74 0.90, "
-                              "59.59 0.97, 59.45 1.03, 59.30 1.10, 59.15 1.16, 59.01 1.23))")
-    mmd_yaml = yaml.load(
-        resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
-    )
-    data = md.get_geographic_extent_polygon(mmd_yaml["geographic_extent"].pop("polygon"), ncin)
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.geospatial_bounds = ("POLYGON ((59.01 1.23, 59.06 1.66, 59.10 2.09, 59.15 2.53, "
+                                  "59.19 2.96, 59.24 3.40, 59.28 3.84, 59.32 4.28, 59.35 4.72, "
+                                  "59.39 5.16, 59.43 5.61, 59.43 5.61, 59.57 5.57, 59.72 5.52, "
+                                  "59.87 5.48, 60.01 5.43, 60.16 5.39, 60.31 5.34, 60.46 5.29, "
+                                  "60.60 5.25, 60.75 5.20, 60.92 5.15, 60.92 5.15, 60.88 4.67, "
+                                  "60.84 4.21, 60.80 3.75, 60.76 3.29, 60.72 2.84, 60.68 2.38, "
+                                  "60.63 1.92, 60.59 1.47, 60.54 1.02, 60.49 0.56, 60.49 0.56, "
+                                  "60.33 0.64, 60.18 0.71, 60.03 0.77, 59.89 0.84, 59.74 0.90, "
+                                  "59.59 0.97, 59.45 1.03, 59.30 1.10, 59.15 1.16, 59.01 1.23))")
+        mmd_yaml = yaml.load(
+            resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
+        )
+        data = md.get_geographic_extent_polygon(mmd_yaml["geographic_extent"].pop("polygon"), ncin)
+
     assert data["srsName"] == "EPSG:4326"
 
 
@@ -163,49 +165,44 @@ def test_get_related_dataset(dataDir):
     )
     # One related dataset
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)'
-    data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)'
+        data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
     assert data[0]['id'] == 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb'
-    ncin.close()
 
     # Two related datasets
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.related_dataset = (
-        'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent), '
-        'no.met:b7cb7934-78ca-4439-812e-f560df3fe7eb (auxiliary)')
-    data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.related_dataset = (
+            'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (parent), '
+            'no.met:b7cb7934-78ca-4439-812e-f560df3fe7eb (auxiliary)')
+        data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
     assert data[0]['id'] == 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb'
     assert data[1]['id'] == 'no.met:b7cb7934-78ca-4439-812e-f560df3fe7eb'
-    ncin.close()
 
     # Malformed relation
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb'
-    data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb'
+        data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
     assert 'The global attribute "related_dataset" is malformed' in \
         md.missing_attributes['errors'][0]
-    ncin.close()
 
     # Invalid relation type
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (child)'
-    data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.related_dataset = 'no.met:b7cb7934-77ca-4439-812e-f560df3fe7eb (child)'
+        data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
     assert 'The dataset relation type must be either' in \
         md.missing_attributes['errors'][0]
-    ncin.close()
 
     # Invalid identifier pattern
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
-    ncin = Dataset(md.netcdf_file, "w", diskless=True)
-    ncin.related_dataset = 'b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)'
-    data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
+    with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+        ncin.related_dataset = 'b7cb7934-77ca-4439-812e-f560df3fe7eb (parent)'
+        data = md.get_related_dataset(mmd_yaml['related_dataset'], ncin)
     assert 'missing naming_authority in the identifier' in \
         md.missing_attributes['errors'][0]
-    ncin.close()
 
 
 @pytest.mark.py_mmd_tools
@@ -234,8 +231,8 @@ def testNc_to_mmd_Get_acdd_metadata(dataDir):
     key = "dataset_production_status"
     test_in = os.path.join(dataDir, "reference_nc.nc")
     md = Nc_to_mmd(test_in, check_only=True)
-    ncin = Dataset(test_in, "w", diskless=True)
-    assert md.get_acdd_metadata(mmd_yaml[key], ncin, key) == "Complete"
+    with Dataset(test_in, "w", diskless=True) as ncin:
+        assert md.get_acdd_metadata(mmd_yaml[key], ncin, key) == "Complete"
 
 
 @pytest.mark.py_mmd_tools
@@ -410,24 +407,23 @@ def test_get_operational_status(dataDir, monkeypatch):
     mmd_element = mmd_yaml["operational_status"]
     test_in = os.path.join(dataDir, "reference_nc.nc")
     md = Nc_to_mmd(test_in, check_only=True)
-    ncin = Dataset(test_in, "w", diskless=True)
-
-    # processing_level is not present
-    value = md.get_operational_status(mmd_element, ncin)
-    assert value == "Not available"
-
-    # processing_level is not valid
-    ncin.processing_level = "kjhhas"
-    mmd_element["maxOccurs"] = "1"
-    value = md.get_operational_status(mmd_element, ncin)
-    assert ("The ACDD attribute 'processing_level' in MMD attribute 'operational_status'"
-            in md.missing_attributes['errors'][0])
+    with Dataset(test_in, "w", diskless=True) as ncin:
+        # processing_level is not present
+        value = md.get_operational_status(mmd_element, ncin)
+        assert value == "Not available"
 
-    # repetition of processing_level is allowed
-    mmd_element["maxOccurs"] = "unbounded"
-    with pytest.raises(ValueError) as ve:
+        # processing_level is not valid
+        ncin.processing_level = "kjhhas"
+        mmd_element["maxOccurs"] = "1"
         value = md.get_operational_status(mmd_element, ncin)
-    assert str(ve.value) == "This is not expected..."
+        assert ("The ACDD attribute 'processing_level' in MMD attribute 'operational_status'"
+                in md.missing_attributes['errors'][0])
+
+        # repetition of processing_level is allowed
+        mmd_element["maxOccurs"] = "unbounded"
+        with pytest.raises(ValueError) as ve:
+            value = md.get_operational_status(mmd_element, ncin)
+        assert str(ve.value) == "This is not expected..."
 
 
 @pytest.mark.py_mmd_tools
@@ -438,24 +434,23 @@ def test_dataset_production_status(dataDir):
     mmd_element = mmd_yaml["dataset_production_status"]
     test_in = os.path.join(dataDir, "reference_nc.nc")
     md = Nc_to_mmd(test_in, check_only=True)
-    ncin = Dataset(test_in, "w", diskless=True)
-
-    # dataset_production_status is not present
-    value = md.get_dataset_production_status(mmd_element, ncin)
-    assert value == "Complete"
-
-    # dataset_production_status is not valid
-    ncin.dataset_production_status = "kjhhas"
-    mmd_element["maxOccurs"] = "1"
-    value = md.get_dataset_production_status(mmd_element, ncin)
-    assert "The ACDD attribute 'dataset_production_status'"
-    "must " in md.missing_attributes['errors'][0]
+    with Dataset(test_in, "w", diskless=True) as ncin:
+        # dataset_production_status is not present
+        value = md.get_dataset_production_status(mmd_element, ncin)
+        assert value == "Complete"
 
-    # repetition of processing_level is allowed
-    mmd_element["maxOccurs"] = "unbounded"
-    with pytest.raises(ValueError) as ve:
+        # dataset_production_status is not valid
+        ncin.dataset_production_status = "kjhhas"
+        mmd_element["maxOccurs"] = "1"
         value = md.get_dataset_production_status(mmd_element, ncin)
-    assert str(ve.value) == "This is not expected..."
+        assert "The ACDD attribute 'dataset_production_status'"
+        "must " in md.missing_attributes['errors'][0]
+
+        # repetition of processing_level is allowed
+        mmd_element["maxOccurs"] = "unbounded"
+        with pytest.raises(ValueError) as ve:
+            value = md.get_dataset_production_status(mmd_element, ncin)
+        assert str(ve.value) == "This is not expected..."
 
 
 @pytest.mark.py_mmd_tools
@@ -466,19 +461,18 @@ def test_get_quality_control(dataDir):
     mmd_element = mmd_yaml["quality_control"]
     test_in = os.path.join(dataDir, "reference_nc.nc")
     md = Nc_to_mmd(test_in, check_only=True)
-    ncin = Dataset(test_in, "w", diskless=True)
-
-    # processing_level is not valid
-    ncin.quality_control = "kjhhas"
-    mmd_element["maxOccurs"] = "1"
-    md.get_quality_control(mmd_element, ncin)
-    assert "The ACDD attribute 'quality_control' must " in md.missing_attributes['errors'][0]
-
-    # repetition of processing_level is allowed
-    mmd_element["maxOccurs"] = "unbounded"
-    with pytest.raises(ValueError) as ve:
+    with Dataset(test_in, "w", diskless=True) as ncin:
+        # processing_level is not valid
+        ncin.quality_control = "kjhhas"
+        mmd_element["maxOccurs"] = "1"
         md.get_quality_control(mmd_element, ncin)
-    assert str(ve.value) == "This is not expected..."
+        assert "The ACDD attribute 'quality_control' must " in md.missing_attributes['errors'][0]
+
+        # repetition of processing_level is allowed
+        mmd_element["maxOccurs"] = "unbounded"
+        with pytest.raises(ValueError) as ve:
+            md.get_quality_control(mmd_element, ncin)
+        assert str(ve.value) == "This is not expected..."
 
 
 @pytest.mark.py_mmd_tools
@@ -914,10 +908,10 @@ def test_license__deprecated_attrs(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "CC-BY-4.0"
-        ncin.license_resource = "http://spdx.org/licenses/CC-BY-4.0"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "CC-BY-4.0"
+            ncin.license_resource = "http://spdx.org/licenses/CC-BY-4.0"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
         self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0')
         self.assertEqual(value['identifier'], 'CC-BY-4.0')
         self.assertEqual(
@@ -929,9 +923,9 @@ def test_license__invalid_url(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "spdx.org/licenses/CC-BY-4.0"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "spdx.org/licenses/CC-BY-4.0"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
         self.assertIsNone(value)
         self.assertEqual(
             md.missing_attributes["errors"][0],
@@ -946,9 +940,9 @@ def test_license__basic(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
         self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0')
         self.assertEqual(value['identifier'], 'CC-BY-4.0')
 
@@ -959,9 +953,10 @@ def test_license__simple(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "http://spdx.org/licenses/CC-BY-4.0"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "http://spdx.org/licenses/CC-BY-4.0"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
+
         # met-vocab-tools should be able to find the license by
         # searching a url (see issue https://github.com/metno/met-vocab-tools/issues/25):
         # self.assertEqual(value['identifier'], 'CC-BY-4.0')
@@ -976,9 +971,10 @@ def test_license__only_url_but_not_standard(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "http://spdx.org/licenses/CC-BY-4.1"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "http://spdx.org/licenses/CC-BY-4.1"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
+
         # met-vocab-tools should be able to find the license by
         # searching a url (see issue https://github.com/metno/met-vocab-tools/issues/25):
         # self.assertEqual(value['identifier'], 'CC-BY-4.0')
@@ -994,9 +990,10 @@ def test_license__according_to_adc1(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "http://spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "http://spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
+
         self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0')
         self.assertEqual(value['identifier'], 'CC-BY-4.0')
 
@@ -1007,9 +1004,10 @@ def test_license__according_to_adc2(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "http://spdx.org/licenses/CC-BY-4.0 (CC-BY-4.0)"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
+
         self.assertEqual(value['resource'], 'http://spdx.org/licenses/CC-BY-4.0')
         self.assertEqual(value['identifier'], 'CC-BY-4.0')
 
@@ -1021,10 +1019,10 @@ def test_license__not_standard(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.license = "https://earth.esa.int/eogateway/documents/20142/1564626/" \
-                       "ESA-Data-Policy-ESA-PB-EO-2010-54.pdf (ESA earth observation data policy)"
-        value = md.get_license(mmd_yaml['use_constraint'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.license = "https://earth.esa.int/eogateway/documents/20142/1564626/" \
+                           "ESA-Data-Policy-ESA-PB-EO-2010-54.pdf (ESA earth observation data policy)"
+            value = md.get_license(mmd_yaml['use_constraint'], ncin)
         self.assertEqual(
             value['license_text'],
             "https://earth.esa.int/eogateway/documents/20142/1564626/"
@@ -1100,11 +1098,11 @@ def test_polygon_is_not_wkt(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.geospatial_bounds = ""
-        md.get_geographic_extent_polygon(
-            mmd_yaml['geographic_extent']['polygon'], ncin
-        )
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.geospatial_bounds = ""
+            md.get_geographic_extent_polygon(
+                mmd_yaml['geographic_extent']['polygon'], ncin
+            )
         self.assertEqual(
             md.missing_attributes["errors"][0],
             "geospatial_bounds must be formatted as a WKT string"
@@ -1149,13 +1147,13 @@ def test_geographic_extent_rectangle_is_floatable(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc.nc'), check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.geospatial_lat_max = "60.158733f; // float"
-        ncin.geospatial_lat_min = "59.78492f; // float"
-        ncin.geospatial_lon_max = "10.944986f; // float"
-        ncin.geospatial_lon_min = "10.508897f; // float"
-        md.get_geographic_extent_rectangle(
-            mmd_yaml['geographic_extent']['rectangle'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.geospatial_lat_max = "60.158733f; // float"
+            ncin.geospatial_lat_min = "59.78492f; // float"
+            ncin.geospatial_lon_max = "10.944986f; // float"
+            ncin.geospatial_lon_min = "10.508897f; // float"
+            md.get_geographic_extent_rectangle(
+                mmd_yaml['geographic_extent']['rectangle'], ncin)
         self.assertEqual(
             md.missing_attributes['errors'][0],
             'geospatial_lat_max must be convertible to float type.')
@@ -1305,12 +1303,11 @@ def test_alternate_identifier_wrong_format(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_with_altID.nc'), check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.alternate_identifier = 'wrong format, missing type'
-        value = md.get_alternate_identifier(
-            mmd_yaml['alternate_identifier'], ncin
-        )
-        print(value)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.alternate_identifier = 'wrong format, missing type'
+            value = md.get_alternate_identifier(
+                mmd_yaml['alternate_identifier'], ncin
+            )
         self.assertEqual(
             md.missing_attributes['errors'][0],
             'alternate_identifier must be formed as <url> (<type>).'
@@ -1465,16 +1462,15 @@ def test_temporal_extent_two_startdates_one_wrong(self):
         )
         md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_attrs_multiple.nc'),
                        check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-
         valid_start = '2020-11-27T13:40:02.019817Z'
         invalid_start = '2020-13-27T13:40:02.019817'  # month outside [0,12]
         valid_end = '2020-11-27T13:51:24.401505Z'
         invalid_end = '2020-13-27T13:51:24.019817'  # month outside [0,12]
 
-        ncin.time_coverage_start = '{}, {}'.format(valid_start, invalid_start)
-        ncin.time_coverage_end = '{}, {}'.format(valid_end, invalid_end)
-        value = md.get_temporal_extents(mmd_yaml['temporal_extent'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.time_coverage_start = '{}, {}'.format(valid_start, invalid_start)
+            ncin.time_coverage_end = '{}, {}'.format(valid_end, invalid_end)
+            value = md.get_temporal_extents(mmd_yaml['temporal_extent'], ncin)
 
         template = 'ACDD start/end datetime {} is not valid ISO8601:'
         self.assertEqual(value[0]['start_date'], valid_start)
@@ -1621,11 +1617,10 @@ def test_get_personnel_role_invalid(self):
         mmd_element = mmd_yaml["personnel"]
         test_in = os.path.abspath('tests/data/reference_nc.nc')
         md = Nc_to_mmd(test_in, check_only=True)
-        ncin = Dataset(test_in, "w", diskless=True)
-
-        # iso_topic_category is not valid
-        ncin.creator_role = "abcd"
-        md.get_personnel(mmd_element, ncin)
+        with Dataset(test_in, "w", diskless=True) as ncin:
+            # iso_topic_category is not valid
+            ncin.creator_role = "abcd"
+            md.get_personnel(mmd_element, ncin)
         assert "The ACDD attribute 'contact_roles' must" in md.missing_attributes['errors'][0]
 
     def test_personnel(self):
@@ -1659,11 +1654,10 @@ def test_get_iso_topic_category_invalid(self):
         mmd_element = mmd_yaml["iso_topic_category"]
         test_in = os.path.abspath('tests/data/reference_nc.nc')
         md = Nc_to_mmd(test_in, check_only=True)
-        ncin = Dataset(test_in, "w", diskless=True)
-
-        # iso_topic_category is not valid
-        ncin.iso_topic_category = "abcd"
-        md.get_iso_topic_category(mmd_element, ncin)
+        with Dataset(test_in, "w", diskless=True) as ncin:
+            # iso_topic_category is not valid
+            ncin.iso_topic_category = "abcd"
+            md.get_iso_topic_category(mmd_element, ncin)
         assert "The ACDD attribute 'iso_topic_category' must" in md.missing_attributes['errors'][0]
 
     def test_get_iso_topic_category_not_available(self):
@@ -1673,10 +1667,9 @@ def test_get_iso_topic_category_not_available(self):
         mmd_element = mmd_yaml["iso_topic_category"]
         test_in = os.path.abspath('tests/data/reference_nc.nc')
         md = Nc_to_mmd(test_in, check_only=True)
-        ncin = Dataset(test_in, "w", diskless=True)
-
-        # iso_topic_category is not present
-        value = md.get_iso_topic_category(mmd_element, ncin)
+        with Dataset(test_in, "w", diskless=True) as ncin:
+            # iso_topic_category is not present
+            value = md.get_iso_topic_category(mmd_element, ncin)
         assert value == ["Not available"]
 
     def test_get_activity_type_invalid(self):
@@ -1689,11 +1682,10 @@ def test_get_activity_type_invalid(self):
         mmd_element = mmd_yaml["activity_type"]
         test_in = os.path.abspath('tests/data/reference_nc.nc')
         md = Nc_to_mmd(test_in, check_only=True)
-        ncin = Dataset(test_in, "w", diskless=True)
-
-        # activity_type is not valid
-        ncin.source = "abcd"
-        md.get_activity_type(mmd_element, ncin)
+        with Dataset(test_in, "w", diskless=True) as ncin:
+            # activity_type is not valid
+            ncin.source = "abcd"
+            md.get_activity_type(mmd_element, ncin)
         assert ("The ACDD attribute 'source' in MMD attribute 'activity_type'"
                 in md.missing_attributes['errors'][0])
 
@@ -1710,12 +1702,12 @@ def test_platform_resource_not_MMD(self):
         # The GCMD resource url
         resource = "https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/platforms"
         # Define dataset
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = "Sentinel-1A"
-        ncin.platform_vocabulary = resource
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = "Sentinel-1A"
+            ncin.platform_vocabulary = resource
 
-        # Get the platform dict
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+            # Get the platform dict
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
         self.assertEqual(value[0]['resource'], resource)
 
     def test_missing_vocabulary_platform_instrument_short_name(self):
@@ -1729,13 +1721,13 @@ def test_missing_vocabulary_platform_instrument_short_name(self):
         md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_missing_keywords_vocab.nc'),
                        check_only=True)
         # Define dataset
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = 'Suomi National Polar-orbiting Partnership (SNPP)'
-        ncin.instrument = 'VIIRS'
-        ncin.instrument_vocabulary = 'not a valid vocab url'
-        ncin.platform_vocabulary = 'https://www.wmo-sat.info/oscar/satellites/view/342'
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = 'Suomi National Polar-orbiting Partnership (SNPP)'
+            ncin.instrument = 'VIIRS'
+            ncin.instrument_vocabulary = 'not a valid vocab url'
+            ncin.platform_vocabulary = 'https://www.wmo-sat.info/oscar/satellites/view/342'
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
 
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
         self.assertEqual(value[0]['resource'],
                          'https://www.wmo-sat.info/oscar/satellites/view/342')
         self.assertEqual(value[0]['short_name'], 'SNPP')
@@ -1752,10 +1744,11 @@ def test_platform_vocabulary_invalid_url(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = 'Environmental Satellite'
-        ncin.platform_vocabulary = 'invalid_url'
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = 'Environmental Satellite'
+            ncin.platform_vocabulary = 'invalid_url'
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
+
         self.assertEqual(value, [])
         self.assertEqual(
             md.missing_attributes['errors'][0],
@@ -1774,9 +1767,9 @@ def test_wrong_platform_name(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(Too Many)(SNPP)'
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(Too Many)(SNPP)'
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
         self.assertEqual(value, [])
         self.assertEqual(
             md.missing_attributes['errors'][0],
@@ -1794,10 +1787,11 @@ def test_wrong_instrument_name(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = "Suomi National Polar-orbiting Partnership (SNPP)"
-        ncin.instrument = 'InstrName (Instrument Name)(Too Many)(IN)'
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = "Suomi National Polar-orbiting Partnership (SNPP)"
+            ncin.instrument = 'InstrName (Instrument Name)(Too Many)(IN)'
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
+
         self.assertEqual(value[0]['short_name'], "SNPP")
         self.assertEqual(
             md.missing_attributes['warnings'][0],
@@ -1816,11 +1810,12 @@ def test_platform_name_extra_parentheses(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(SNPP)'
-        # A valid url is required
-        ncin.platform_vocabulary = "https://www.validurl.no"
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = 'Suomi National Polar-orbiting Partnership (Suomi NPP)(SNPP)'
+            # A valid url is required
+            ncin.platform_vocabulary = "https://www.validurl.no"
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
+
         self.assertEqual(value[0]['long_name'],
                          'Suomi National Polar-orbiting Partnership (Suomi NPP)')
         self.assertEqual(value[0]['short_name'], 'SNPP')
@@ -1832,12 +1827,13 @@ def test_instrument_name_extra_parentheses(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.platform = "Platform Name (PN)"
-        ncin.platform_vocabulary = "https://www.validurl.no"
-        # This is an entry in MMD:
-        ncin.instrument = 'Synthetic Aperture Radar (C-band) (SAR-C)'
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.platform = "Platform Name (PN)"
+            ncin.platform_vocabulary = "https://www.validurl.no"
+            # This is an entry in MMD:
+            ncin.instrument = 'Synthetic Aperture Radar (C-band) (SAR-C)'
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
+
         self.assertEqual(value[0]['instrument']['short_name'], 'SAR-C')
         self.assertEqual(value[0]['instrument']['long_name'], 'Synthetic Aperture Radar (C-band)')
 
@@ -1850,10 +1846,11 @@ def test_missing_platform_vocabulary(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        # Note that Envisat is not in the MMD vocabulary
-        ncin.platform = 'Envisat'
-        value = md.get_platforms(mmd_yaml['platform'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            # Note that Envisat is not in the MMD vocabulary
+            ncin.platform = 'Envisat'
+            value = md.get_platforms(mmd_yaml['platform'], ncin)
+
         self.assertEqual(value, [])
         self.assertEqual(
             md.missing_attributes['errors'][0],
@@ -1922,13 +1919,14 @@ def test__keywords_vocabulary__correctly_formatted(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \
-                        "GEMET:Meteorological geographical features, " \
-                        "GEMET:Atmospheric conditions, " \
-                        "NORTHEMES:Weather and climate"
-        ncin.keywords_vocabulary = ""
-        md.get_keywords(mmd_yaml['keywords'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \
+                            "GEMET:Meteorological geographical features, " \
+                            "GEMET:Atmospheric conditions, " \
+                            "NORTHEMES:Weather and climate"
+            ncin.keywords_vocabulary = ""
+            md.get_keywords(mmd_yaml['keywords'], ncin)
+
         self.assertEqual(
             md.missing_attributes['errors'][0],
             'keywords_vocabulary must be formatted as <short_name>:<long_name>:<url>'
@@ -1942,18 +1940,19 @@ def test_keywords_vocabulary__invalid_url_pattern(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \
-                        "GEMET:Meteorological geographical features, " \
-                        "GEMET:Atmospheric conditions, " \
-                        "NORTHEMES:Weather and climate"
-        ncin.keywords_vocabulary = (
-            "GCMDSK:GCMD Science Keywords:"
-            "https://gcmd.earth_data.nasa.gov/kms/concepts/concept_scheme/sciencekeywords, "
-            "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, "
-            "NORTHEMES:GeoNorge Themes:"
-            "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling")
-        md.get_keywords(mmd_yaml['keywords'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.keywords = "GCMDSK:Earth Science > Atmosphere > Atmospheric radiation, " \
+                            "GEMET:Meteorological geographical features, " \
+                            "GEMET:Atmospheric conditions, " \
+                            "NORTHEMES:Weather and climate"
+            ncin.keywords_vocabulary = (
+                "GCMDSK:GCMD Science Keywords:"
+                "https://gcmd.earth_data.nasa.gov/kms/concepts/concept_scheme/sciencekeywords, "
+                "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, "
+                "NORTHEMES:GeoNorge Themes:"
+                "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling")
+            md.get_keywords(mmd_yaml['keywords'], ncin)
+
         self.assertEqual(
             md.missing_attributes['errors'][0],
             'https://gcmd.earth_data.nasa.gov/kms/concepts/concept_scheme/sciencekeywords'
@@ -1970,17 +1969,18 @@ def test_spaces_around_keywords_are_stripped(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.keywords = "GCMDSK: Earth Science > Atmosphere > Atmospheric radiation , " \
-                        "GEMET: Meteorological geographical features, " \
-                        "GEMET: Atmospheric conditions , " \
-                        "NORTHEMES:Weather and climate "
-        ncin.keywords_vocabulary = (
-            "GCMDSK:GCMD Science Keywords:https://vocab.met.no/GCMDSK,"
-            "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, "
-            "NORTHEMES:GeoNorge Themes:"
-            "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling")
-        data = md.get_keywords(mmd_yaml['keywords'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.keywords = "GCMDSK: Earth Science > Atmosphere > Atmospheric radiation , " \
+                            "GEMET: Meteorological geographical features, " \
+                            "GEMET: Atmospheric conditions , " \
+                            "NORTHEMES:Weather and climate "
+            ncin.keywords_vocabulary = (
+                "GCMDSK:GCMD Science Keywords:https://vocab.met.no/GCMDSK,"
+                "GEMET:INSPIRE Themes:http://inspire.ec.eur_opa.eu/theme, "
+                "NORTHEMES:GeoNorge Themes:"
+                "https://register.geonorge.no/metadata-kodelister/nasjonal-temainndeling")
+            data = md.get_keywords(mmd_yaml['keywords'], ncin)
+
         self.assertEqual(data[0]["keyword"][0],
                          "Earth Science > Atmosphere > Atmospheric radiation")
 
@@ -2176,13 +2176,14 @@ def test_dataset_citation_as_kwarg(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.creator_name = "Tester Test"
-        ncin.date_created = "2022-11-04T11:06:10Z"
-        ncin.title = "Test dataset"
-        ncin.publisher_name = "Norwegian Meteorological Institute"
-        ncin.metadata_link = "invalid_url"
-        value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin, dataset_citation=dc)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.creator_name = "Tester Test"
+            ncin.date_created = "2022-11-04T11:06:10Z"
+            ncin.title = "Test dataset"
+            ncin.publisher_name = "Norwegian Meteorological Institute"
+            ncin.metadata_link = "invalid_url"
+            value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin, dataset_citation=dc)
+
         self.assertEqual(value[0]['author'], 'No Name')
 
     def test_check_only(self):
@@ -2229,24 +2230,26 @@ def test_dataset_citation_invalid_date(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc.nc'), check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.time_coverage_start = "2020-11-27T13:40:02.019817Z"
-        ncin.time_coverage_end = "2020-11-27T13:51:24.401505Z"
-        ncin.creator_name = 'Kreator Kreatorsen'
-        ncin.date_created = "2020-11-28T13:51:24.401505Z"
-        ncin.title = "Test dataset"
-        ncin.metadata_link = "https://data.met.no/dataset/uuid-for-the-dataset"
-        ncin.references = "Some free-text refences"
-        value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin)
-        self.assertEqual(value[0]['author'], 'Kreator Kreatorsen')
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.time_coverage_start = "2020-11-27T13:40:02.019817Z"
+            ncin.time_coverage_end = "2020-11-27T13:51:24.401505Z"
+            ncin.creator_name = 'Kreator Kreatorsen'
+            ncin.date_created = "2020-11-28T13:51:24.401505Z"
+            ncin.title = "Test dataset"
+            ncin.metadata_link = "https://data.met.no/dataset/uuid-for-the-dataset"
+            ncin.references = "Some free-text refences"
+            value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin)
+
+            self.assertEqual(value[0]['author'], 'Kreator Kreatorsen')
+
+            # Test that an error is appended if date created is not in
+            # the correct format
+            ncin.date_created = "2020-99-28 13:51:24"
+            mmd_yaml = yaml.load(
+                resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
+            )
+            value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin)
 
-        # Test that an error is appended if date created is not in
-        # the correct format
-        ncin.date_created = "2020-99-28 13:51:24"
-        mmd_yaml = yaml.load(
-            resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
-        )
-        value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin)
         self.assertIn(
             "ACDD attribute date_created contains an invalid ISO8601 date:",
             md.missing_attributes['errors'][0]
@@ -2522,7 +2525,7 @@ def test_create_mmd_missing_update_times(self):
         )
 
     def test_publication_date__is_a_list_of_dates(self):
-        """The publication data can be a list of dates but is set to
+        """The publication date can be a list of dates but is set to
         date_created. In most cases it is the only one item, but we
         need to check that what we get from the netcdf file is an
         actual list, and that the items are actual datestrings.
@@ -2547,9 +2550,10 @@ def test_get_metadata_updates__datetimes_not_iso(self):
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
         # To overwrite date_created, wihtout saving it to file we use diskless
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.date_created = '2019/01/01 00:00:00'
-        md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.date_created = '2019/01/01 00:00:00'
+            md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin)
+
         self.assertEqual(
             md.missing_attributes['errors'][0],
             "Datetime element must be in ISO8601 format: "
@@ -2563,13 +2567,14 @@ def test_ACDD_attr__date_created(self):
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
         # To overwrite date_created, wihtout saving it to file we use diskless
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.date_created = '2019-01-01T00:00:00Z'
-        data = md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin)
-        self.assertIn(
-            {'datetime':  ncin.date_created, 'type': 'Created'},
-            data['update']
-        )
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.date_created = '2019-01-01T00:00:00Z'
+            data = md.get_metadata_updates(mmd_yaml['last_metadata_update'], ncin)
+
+            self.assertIn(
+                {'datetime':  ncin.date_created, 'type': 'Created'},
+                data['update']
+            )
         assert (len(data['update']) == 1)
         self.assertEqual(data['update'][0]['type'], 'Created')
 
@@ -2670,10 +2675,11 @@ def test_access_constraint(self):
 
     def test_check_attributes_not_empty(self):
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.geospatial_bounds = ""
-        with self.assertRaises(ValueError) as e:
-            md.check_attributes_not_empty(ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.geospatial_bounds = ""
+            with self.assertRaises(ValueError) as e:
+                md.check_attributes_not_empty(ncin)
+
         self.assertIn('Global attribute geospatial_bounds is empty - please correct.',
                       str(e.exception))
 
@@ -2683,18 +2689,19 @@ def test_check_attributes_not_empty__accepts_0(self):
         a bug. This test checks that 0 is accepted.
         """
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.subswath = 0
-        self.assertEqual(None, md.check_attributes_not_empty(ncin))
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.subswath = 0
+            self.assertEqual(None, md.check_attributes_not_empty(ncin))
 
     def test_check_feature_type__missing(self):
         """ Test that the correct warning is issued if the CF
         attribute featureType is missing.
         """
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.tull = "tull"
-        md.check_feature_type(ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.tull = "tull"
+            md.check_feature_type(ncin)
+
         self.assertEqual(
             md.missing_attributes["warnings"][0],
             "CF attribute featureType is missing - one of the feature"
@@ -2707,9 +2714,10 @@ def test_check_feature_type__wrong(self):
         attribute featureType is wrong.
         """
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.featureType = "tull"
-        md.check_feature_type(ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.featureType = "tull"
+            md.check_feature_type(ncin)
+
         self.assertEqual(
             md.missing_attributes["errors"][0],
             "featureType seems to be wrong - it should be picked from "
@@ -2719,9 +2727,10 @@ def test_check_feature_type__wrong(self):
 
     def test_check_conventions__missing(self):
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.tull = "tull"
-        md.check_conventions(ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.tull = "tull"
+            md.check_conventions(ncin)
+
         self.assertEqual(
             md.missing_attributes["errors"][0],
             'Required attribute "Conventions" is missing. This should'
@@ -2730,9 +2739,10 @@ def test_check_conventions__missing(self):
 
     def test_check_conventions__cf_and_acdd_missing(self):
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.Conventions = "tull"
-        md.check_conventions(ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.Conventions = "tull"
+            md.check_conventions(ncin)
+
         self.assertEqual(
             md.missing_attributes["errors"][0],
             'The dataset should follow the CF-standard. Please '
@@ -2752,9 +2762,10 @@ def test_institution_name_parsing(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.institution = "Norwegian Meteorological Institute (MET Norway)"
-        data = md.get_data_centers(mmd_yaml['data_center'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.institution = "Norwegian Meteorological Institute (MET Norway)"
+            data = md.get_data_centers(mmd_yaml['data_center'], ncin)
+
         self.assertEqual(data[0]['data_center_name']['long_name'],
                          'Norwegian Meteorological Institute')
         self.assertEqual(data[0]['data_center_name']['short_name'], 'MET Norway')
@@ -2765,9 +2776,10 @@ def test_institution_short_name_missing(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.institution = "Norwegian Meteorological Institute"
-        md.get_data_centers(mmd_yaml['data_center'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.institution = "Norwegian Meteorological Institute"
+            md.get_data_centers(mmd_yaml['data_center'], ncin)
+
         self.assertEqual(
             md.missing_attributes['errors'][0],
             "institution must be formed as <institution long name> (<institution short name>). "
@@ -2794,13 +2806,14 @@ def test_acdd_references_as_related_information2(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.references = (
-            "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
-            " (Dataset landing page),"  # added a space
-            "https://ieeexplore.ieee.org/document/7914752 (Scientific publication)"  # added space
-        )
-        data = md.get_related_information(mmd_yaml['related_information'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.references = (
+                "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
+                " (Dataset landing page),"  # added a space
+                "https://ieeexplore.ieee.org/document/7914752 (Scientific publication)"  # added space
+            )
+            data = md.get_related_information(mmd_yaml['related_information'], ncin)
+
         # Note: A landing page url created from the netcdf id and
         # naming_authority attributes overrides the references information
         self.assertEqual(
@@ -2819,13 +2832,14 @@ def test_acdd_references_invalid_type(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.references = (
-            "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
-            "(kjhf),"
-            "https://ieeexplore.ieee.org/document/7914752(Scientific publication)"
-        )
-        data = md.get_related_information(mmd_yaml['related_information'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.references = (
+                "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
+                "(kjhf),"
+                "https://ieeexplore.ieee.org/document/7914752(Scientific publication)"
+            )
+            data = md.get_related_information(mmd_yaml['related_information'], ncin)
+
         self.assertEqual(data[1]['type'], 'Scientific publication')
         self.assertEqual(
             md.missing_attributes["errors"][0],
@@ -2842,13 +2856,14 @@ def test_acdd_references_invalid_url(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.references = (
-            "https://data.invalid_domain.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
-            "(Dataset landing page),"
-            "https://ieeexplore.ieee__.org/document/7914752(Scientific publication)"
-        )
-        md.get_related_information(mmd_yaml['related_information'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.references = (
+                "https://data.invalid_domain.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
+                "(Dataset landing page),"
+                "https://ieeexplore.ieee__.org/document/7914752(Scientific publication)"
+            )
+            md.get_related_information(mmd_yaml['related_information'], ncin)
+
         self.assertEqual(
             md.missing_attributes["errors"][0],
             'references must contain valid uris')
@@ -2864,9 +2879,10 @@ def test_acdd_references_malformed(self):
             resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
         )
         md = Nc_to_mmd(self.fail_nc, check_only=True)
-        ncin = Dataset(md.netcdf_file, "w", diskless=True)
-        ncin.references = "landing_page, paper"
-        md.get_related_information(mmd_yaml['related_information'], ncin)
+        with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
+            ncin.references = "landing_page, paper"
+            md.get_related_information(mmd_yaml['related_information'], ncin)
+
         self.assertEqual(
             md.missing_attributes["errors"][0],
             "references must be formed as <uri>(<type>).")

From 36fac4baaa16f1dc2a0a1eb1f42a386ca683939b Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 30 Sep 2024 13:49:34 +0200
Subject: [PATCH 19/32] 337: remove test, as docstring does not make sense

---
 tests/test_nc_to_mmd.py | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/tests/test_nc_to_mmd.py b/tests/test_nc_to_mmd.py
index da5590c1..42c0db1b 100644
--- a/tests/test_nc_to_mmd.py
+++ b/tests/test_nc_to_mmd.py
@@ -2524,23 +2524,6 @@ def test_create_mmd_missing_update_times(self):
             'ACDD attribute date_created is required'
         )
 
-    def test_publication_date__is_a_list_of_dates(self):
-        """The publication date can be a list of dates but is set to
-        date_created. In most cases it is the only one item, but we
-        need to check that what we get from the netcdf file is an
-        actual list, and that the items are actual datestrings.
-        """
-        mmd_yaml = yaml.load(
-            resource_string('py_mmd_tools', 'mmd_elements.yaml'), Loader=yaml.FullLoader
-        )
-        md = Nc_to_mmd(self.reference_nc, check_only=True)
-        # To overwrite date_created, wihtout saving it to file we use diskless
-        ncin = Dataset(md.netcdf_file)
-        data = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin)
-        self.assertEqual(
-            data[0]['title'],
-            'Direct Broadcast data processed in satellite swath to L1C.')
-
     def test_get_metadata_updates__datetimes_not_iso(self):
         """ Test that an error is raised if datetimes of metadata
         updates are not ISO 8601.

From c703e1633773916aed5ce180bf1eab187ff75aeb Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Mon, 30 Sep 2024 16:59:12 +0200
Subject: [PATCH 20/32] #337: fix flake errors

---
 tests/test_nc_to_mmd.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/tests/test_nc_to_mmd.py b/tests/test_nc_to_mmd.py
index 2afa8dfc..7c552e92 100644
--- a/tests/test_nc_to_mmd.py
+++ b/tests/test_nc_to_mmd.py
@@ -121,11 +121,11 @@ def test_separate_repeated(dataDir):
     """
     md = Nc_to_mmd(os.path.join(dataDir, 'reference_nc.nc'), check_only=True)
     with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
-        ncin.platform = ['&quot;Basis&quot;: &quot;Space-based Platforms&quot;',
-                        ' &quot;Category&quot;: &quot;Earth Observation Satellites&quot;',
-                        ' &quot;Sub_Category&quot;: &quot;Sentinel-1&quot;',
-                        ' &quot;Short_Name&quot;: &quot;Sentinel-1A&quot;',
-                        ' &quot;Long_Name&quot;: &quot;Sentinel-1A&quot;']
+        ncin.platform = ["&quot;Basis&quot;: &quot;Space-based Platforms&quot;",
+                         " &quot;Category&quot;: &quot;Earth Observation Satellites&quot;",
+                         " &quot;Sub_Category&quot;: &quot;Sentinel-1&quot;",
+                         " &quot;Short_Name&quot;: &quot;Sentinel-1A&quot;",
+                         " &quot;Long_Name&quot;: &quot;Sentinel-1A&quot;"]
         with pytest.raises(AttributeError) as ee:
             md.separate_repeated(True, getattr(ncin, "platform"))
 
@@ -1020,8 +1020,9 @@ def test_license__not_standard(self):
         )
         md = Nc_to_mmd(self.reference_nc, check_only=True)
         with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
-            ncin.license = "https://earth.esa.int/eogateway/documents/20142/1564626/" \
-                           "ESA-Data-Policy-ESA-PB-EO-2010-54.pdf (ESA earth observation data policy)"
+            ncin.license = ("https://earth.esa.int/eogateway/documents/20142/1564626/"
+                            "ESA-Data-Policy-ESA-PB-EO-2010-54.pdf (ESA earth "
+                            "observation data policy)")
             value = md.get_license(mmd_yaml['use_constraint'], ncin)
         self.assertEqual(
             value['license_text'],
@@ -1305,9 +1306,7 @@ def test_alternate_identifier_wrong_format(self):
         md = Nc_to_mmd(os.path.abspath('tests/data/reference_nc_with_altID.nc'), check_only=True)
         with Dataset(md.netcdf_file, "w", diskless=True) as ncin:
             ncin.alternate_identifier = 'wrong format, missing type'
-            value = md.get_alternate_identifier(
-                mmd_yaml['alternate_identifier'], ncin
-            )
+            md.get_alternate_identifier(mmd_yaml['alternate_identifier'], ncin)
         self.assertEqual(
             md.missing_attributes['errors'][0],
             'alternate_identifier must be formed as <url> (<type>).'
@@ -2181,7 +2180,8 @@ def test_dataset_citation_as_kwarg(self):
             ncin.title = "Test dataset"
             ncin.publisher_name = "Norwegian Meteorological Institute"
             ncin.metadata_link = "invalid_url"
-            value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin, dataset_citation=dc)
+            value = md.get_dataset_citations(mmd_yaml['dataset_citation'], ncin,
+                                             dataset_citation=dc)
 
         self.assertEqual(value[0]['author'], 'No Name')
 
@@ -2792,7 +2792,8 @@ def test_acdd_references_as_related_information2(self):
             ncin.references = (
                 "https://data.met.no/dataset/3f9974bf-b073-4c16-81d8-c34fcf3b1f01"
                 " (Dataset landing page),"  # added a space
-                "https://ieeexplore.ieee.org/document/7914752 (Scientific publication)"  # added space
+                "https://ieeexplore.ieee.org/document/7914752 (Scientific "
+                "publication)"  # added space
             )
             data = md.get_related_information(mmd_yaml['related_information'], ncin)
 

From 65156961bde7e62c7d4e946ba4ce010101d71d48 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 13:54:06 +0200
Subject: [PATCH 21/32] #337: change use of base folders and pattern, update
 tests, and change to raising exceptions if something fails instead of just
 adding failing files to a list

---
 py_mmd_tools/mmd_operations.py        |  62 +++++++--------
 py_mmd_tools/script/move_data.py      |  22 +++---
 tests/data/2024/09/01/reference_nc.nc | Bin 0 -> 30030 bytes
 tests/test_mmd_operations.py          | 109 +++++++++++++++++---------
 tests/test_move_data_script.py        | 109 ++++++++++++++++++++++++--
 5 files changed, 212 insertions(+), 90 deletions(-)
 create mode 100644 tests/data/2024/09/01/reference_nc.nc

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 218d6b19..c8abe0bc 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -16,6 +16,7 @@
 import datetime
 import requests
 import tempfile
+import warnings
 import datetime_glob
 
 
@@ -68,7 +69,7 @@ def mmd_change_file_location(mmd, new_file_location, copy=True):
     indicating if it has been changed or not.
     """
     if not os.path.isfile(mmd):
-        return None, False
+        raise ValueError(f"File does not exist: {mmd}")
     if copy:
         tmp_path = tempfile.gettempdir()
         shutil.copy2(mmd, tmp_path)
@@ -113,8 +114,8 @@ def move_data_file(nc_file, nfl, emsg=""):
     return nc_moved, emsg
 
 
-def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pattern_or_exact,
-              dry_run=True, env="prod"):
+def move_data(mmd_repository_path, old_file_location_base, new_file_location_base,
+              ext_pattern=None, dry_run=True, env="prod"):
     """Update MMD and move data file.
     """
     if env not in ["dev", "staging", "prod"]:
@@ -140,19 +141,16 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
         }
     }
 
-    if os.path.isfile(existing_pathname_pattern_or_exact):
-        existing = [existing_pathname_pattern_or_exact]
-        existing_pathname_pattern = None
+    if os.path.isfile(old_file_location_base):
+        existing = [old_file_location_base]
     else:
         existing = [str(nc_file) for match, nc_file in
-                    datetime_glob.walk(pattern=existing_pathname_pattern_or_exact)]
-        existing_pathname_pattern = existing_pathname_pattern_or_exact
+                    datetime_glob.walk(pattern=os.path.join(old_file_location_base, ext_pattern))]
 
     copy_mmd = True
     if dry_run:
         # Not copying the MMD file will make it easy to check changes
         # with git diff
-        existing_pathname_pattern = None
         copy_mmd = False
 
     updated = []
@@ -160,7 +158,7 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
     for nc_file in existing:
         # Error message
         emsg = ""
-        nfl = new_file_location(nc_file, new_file_location_base, existing_pathname_pattern)
+        nfl = new_file_location(nc_file, new_file_location_base, old_file_location_base)
         try:
             mmd_orig = get_local_mmd_git_path(nc_file, mmd_repository_path)
         except Exception as e:
@@ -171,20 +169,17 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
         remove_file_allowed = os.access(nc_file, os.W_OK)
         write_file_allowed = os.access(nfl, os.W_OK)
         if not remove_file_allowed or not write_file_allowed:
+            if not remove_file_allowed and not write_file_allowed:
+                raise PermissionError(f"Missing permissions to delete {nc_file} "
+                                      f"and to write {nfl}")
             if not remove_file_allowed:
-                not_updated[mmd_orig] = f"Missing permission to delete {nc_file}"
+                raise PermissionError(f"Missing permission to delete {nc_file}")
             if not write_file_allowed:
-                not_updated[mmd_orig] = f"Missing permission to write {nfl}"
-            if not remove_file_allowed and not write_file_allowed:
-                not_updated[mmd_orig] = (f"Missing permission to delete {nc_file} "
-                                         f"nor to write {nfl}")
-            continue
+                raise PermissionError(f"Missing permission to write {nfl}")
 
         mmd_new, mmd_updated = mmd_change_file_location(mmd_orig, nfl, copy=copy_mmd)
         if not mmd_updated:
-            emsg = f"Could not update MMD file for {nc_file}"
-            not_updated[mmd_orig] = emsg
-            continue
+            raise Exception(f"Could not update MMD file for {nc_file}")
 
         # Get MMD content as binary data
         with open(mmd_new, "rb") as fn:
@@ -197,16 +192,15 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
             # be careful with this...
             res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data)
         if res.status_code == 200:
-            # This intentionally becomes True in case of dry-run and
-            # a valid xml
+            # This should be the case for a dry-run and a valid xml
             dmci_updated = True
         else:
-            emsg = "Could not push updated MMD file to the DMCI API."
-            not_updated[mmd_orig] = emsg
-            continue
+            raise Exception(f"Could not push updated MMD file to the DMCI API: {mmd_new}")
 
         if dmci_updated and not dry_run:
             nc_moved, emsg = move_data_file(nc_file, nfl)
+            if not nc_moved:
+                raise Exception(f"Could not move {nc_file} to {nfl}.")
         elif dmci_updated and dry_run:
             nc_moved = True
 
@@ -216,6 +210,9 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
         else:
             ds_found_and_accessible = True
 
+        if not ds_found_and_accessible:
+            warnings.warn(f"Could not find data in CSW catalog: {ds_id}")
+
         if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]):
             updated.append(mmd_orig)
         else:
@@ -224,18 +221,13 @@ def move_data(mmd_repository_path, new_file_location_base, existing_pathname_pat
     return not_updated, updated
 
 
-def new_file_location(nc_file, new_base_loc, existing_pathname_pattern=None):
+def new_file_location(nc_file, new_base_loc, existing_base_loc):
     """Return the name of the new folder where the netcdf file will be
-    stored. If existing_pathname_pattern is None, the returned path
-    will equal the provided parameter new_base_loc. This is to allow
-    usage flexibility of the move_data function.
+    stored. Subfolders of new_base_loc will be created.
     """
-    if existing_pathname_pattern is None:
-        file_path = os.path.join(new_base_loc, os.path.basename(nc_file))
-    else:
-        prefix = re.split(r"[^\w/-]", existing_pathname_pattern)[0]
-        file_path = os.path.join(new_base_loc, nc_file[len(prefix):])
+    if not os.path.isdir(new_base_loc):
+        raise ValueError(f"Folder does not exist: {new_base_loc}")
+    file_path = nc_file.replace(existing_base_loc, new_base_loc)
     new_folder = os.path.dirname(os.path.abspath(file_path))
-    if not os.path.isdir(new_folder):
-        raise ValueError(f"Folder does not exist: {file_path}")
+    os.makedirs(new_folder)
     return new_folder
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index 9a0e83ca..81f470b1 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -28,14 +28,17 @@ def create_parser():
     parser.add_argument(
         "mmd_repository_path", type=str,
         help="Local folder containing all MMD files.")
+    parser.add_argument(
+        "old_file_location_base", type=str,
+        help="Base folder from which the data file(s) will be moved, or exact path to a file.")
     parser.add_argument(
         "new_file_location_base", type=str,
-        help="Base or exact path to the folder to which the new file will be moved.")
+        help="Base or exact path to the folder to which the data file(s) will be moved.")
     parser.add_argument(
-        "existing_pathname_pattern", type=str,
-        help="Pathname pattern to existing file location(s). Allows "
-             "parsing date/times from a path given a glob pattern "
-             "intertwined with date/time format akin to "
+        "--ext_pattern", type=str, default=None,
+        help="Pathname pattern extending old_file_location_base, i.e., extending the "
+             "existing file *base* location(s). Allows parsing date/times from a path "
+             "given a glob pattern intertwined with date/time format akin to "
              "strptime/strftime format.")
     parser.add_argument(
         '--dmci-update', action='store_true',
@@ -55,13 +58,12 @@ def main(args=None):
         raise ValueError(f"Invalid input: {args.new_file_location_base}")
 
     not_updated, updated = move_data(args.mmd_repository_path,
+                                     args.old_file_location_base,
                                      args.new_file_location_base,
-                                     args.existing_pathname_pattern,
+                                     args.ext_pattern,
                                      dry_run=not args.dmci_update)
-    print(f"Updated: {len(updated)}")
-    print(f"Not updated: {len(not_updated)}\n")
-    for key, val in not_updated.items():
-        print(f"{key}: {val}")
+
+    return updated, not_updated
 
 
 def _main():  # pragma: no cover
diff --git a/tests/data/2024/09/01/reference_nc.nc b/tests/data/2024/09/01/reference_nc.nc
new file mode 100644
index 0000000000000000000000000000000000000000..9e053c5c5bf396d8e577fc26da4925234f0d5b2a
GIT binary patch
literal 30030
zcmeHQ3v^vonf_1uN=pi9fzZ|pr}D6c<i1}CuOv4~8)%b~gmx-Aw>S5mB)7S_C){&y
z+ElRB!CA^nKpb7lz><+cfu-UDnT}S#B{Bj_)HO38Q_C70UIU1Tt11HX{d=E%?>Wg$
znj+$ecdy>;bN1Q$-~ayqz5mzV=k((nI=YrFS+#^X)z;E{su!2)&l`5?DasOcubyvL
z7c=C&8#?-GmKJJi-Xki^TWHr)aHi%<YDnWM$FBw|MDupi6E(6)exA%fW9JKx<9m&3
ziKI=|+o<!CMR+0r;hDB8woKMp*dU+SAZgwvpMU#u``NqXGaEeBW|M_Xp_n!<SY=mO
zEi0%V7iZ1fu&D72nsQobgIFMMsTSZ9DRX@Camy|lQ8?N6)$C75fwB5ueYmS{!)B^Q
zCpFHAt!CZJY1#dL#q_;4I&1CRC-aznd$<f!L(L93jcG0N#t+DL^Qr5qh2Lu**w6ut
zFa?+4S$b_fV~3=X>Sbv3nr2aJG1pouYt>Dyb?m<+zT4FgSTu3Gd}e2)a#_kKZKhqo
zg!|z1i$Rgmi!v+aeaV9lwhwIV0(K26@N7#BZ9?|U(?=8C&*S<V#u<y?by`4I9D}Pq
zqW2?PfBzi|=q_H`DI#w9p1t2&OegX(0D5s`OV_+>3B9ruSHB?|Kz7xpTN<dA*P#b;
z2iY^>i%umhPof0>A<701G|?h%az0TbvfVchG}9mh{S{$EM3>)v?OBxN)fv<%iR?Q)
z*PTt)B3y-tE<pA*|5I_=ycky)@)~5ny}h@S+R!C!CCVc!G_2~P9~_0N(*-K@;)Pds
zQ!5h+Bje8YJoVW=`WnMH69Yr`!Yh{z&;u+n&qI4;uXWT9(y3e!Avy)w7e+f$bO{3m
z>SrK3@cfM_8skX@5`vm^@yF9?x|4?w0&&IgXEFxe!K2+kVj_I2ZQmsAI~D~1x*gfE
z%lbb^|IUa7iHgXETDD$7S97;5M5iHpFctVPeV=<l0ZW|U4qo#STFLZyA5kx|3qJDw
z%f9`_-5AJBN$`KMixHwT?v(qcpYk8T=Wb}JjSf!^7t!bje$ot)$WNSnrU;MvTq33@
z75U~YA|8c{9^~N;P>o6AA{{kwZ=fSjK+4%SLi9T)DfQd)@dU2soSbwkQ)C^HklN1K
z;bOCkqjV8t2pXKq(40i4Q<#02QM|rR%fJ2$uJD(e+6TIOupXU^{qctiSJev+6RoZn
zoC58xIx%+tU+yrgn9o1~TmJX)vcnq7B1@$&zs|MJ9$RtCXQ_dOHsn8+DAiu{rO&wp
z!^7U~|3shX8d0KtWcT*{!!5LmS8R-kmFm;JrrTYjU~qZ&9rOiWtmt+a*$d6>chik0
z;i`=&gY5e+c=x$v(REku*hhW5{7x7D=!V?w`{*uS){y^Okp1u*Pv1)oyfV*NNEaj9
zed)<RaEX@BU;5|+^eAJBCg_<qj~)L*I)(Ar3lWR#+2Qv;NDZuhAwhu(UG|dq5i+>o
zX6RmIFJ8RyCp5w^An<ySt&Uy)s7o$nKDp+gOKt_f+IG+-Bl{Nr^D8bPGkV67LoONC
z{_~p;Q6G~Ky_}8gqVu0T<PxB_zqI^SmvDRS+9zJUEq(9ns65vs*8gG`RqKTehhnd1
z&4qN*6+de}C@-sj-(eK6^}o}2aT#GQTgkPeUDk_hqPk>-Ybm*GM3>R&T2C$;#$^J!
ztUZ^+aao%#q2OBXE<4s`Z@DZ>mqc)hDVK;Mr`&W29+%PSGSXb8q|3l_37Ok|xv1-H
zm}}BsU#)2?xHC;-GtM^locVEV3@%%9lI`y3bV4nium?33)bf*>ni|oNJ@WZ<pO;|}
z-6yk>Z@-X~A2v0wV-*2F3t@`XQhZ}uKP!=A@nbtr;sn#VdsAmme|K+BqCeiYv6Jey
z^D_-rJGbg-+ti85qE#z$)oNfQW9j&p)N@85-qEF}Gh;^1%9uHQuvpM@W}*COBAL%5
z2eXDpR|zH{J^%7iuUDoM`wN^@KJSR=beS0SJA+PPP~lKVdz+-wH)}Lj!lcP*A)N;&
zo%kWr6uRjZ=2smvlINGSNME{SzL;_9u2w{I=4t!8B}DiZBB^*%J*~0b<FDKKV|GqU
z*%zs%Dg{auIOp!?ub4f6bQfEnNN=b~#PW#~TZ5eJToud0ng3!M%bAQSQebfa>6Rwh
zLZj5JpKfplkO%>!3mW9p`cj1Jp&$M3Yt8_Ys3inC3s8m2Px2#OGJy2o!kOn7nM^9T
zZ-J6KfV9`f=Ty6b%76W-3s=|<#CeQ8A{V@(H|$=pluqJ;Gvx3#eo?Qdqq)=V(!sBN
z{<#xm%wZ!6kzTp{q*Lfqe6>NMm45KkK01*R>y)8`2X=Y;=m6Jfm-;EO_?T^UC-;IT
z?BPFf!f#AUaT64<+xnji@7zs$xrSc`Chpz3;cEIF7hnX;sr$Eize+D~lQl$a;Zx?u
zJ2`eFe1ER^xx(swlw;Hp17Umk!NjHiCQbY8G9;1u<h9Sz?F^$`hIXF*<f2z%S3dg&
zO8yFxZ9s}SX*1JBKo?hC40KHqa!w!@(_B1uanChBT%33D!_8plG;}f6#njm9UHAMI
zFz+oGqEz@-Xm6LomzzFk*(T=S+aplwe!<KprkI=>g0IIRWS!{BWV2h6*`gr<EURF8
z@DpVavnb%Ca-w_o)UD-vWq#(8V*(Z%tt?tpO|`4UCNA%F9gU~8WEGORbTXe#*v)2a
z1<5-Pt)yXG%w-A|U~_E_*r-0z#&F&+tVGVt7e<I~=>O<Fk}X%h&1}gbF3Rc8$`(X=
zE3;)WFjFKe?d8=$H_XnL%FP;QVhpPh)r(?`k+iUJ@X?1B>FeXY8^u#wE32A3WjLul
zP1Yxzrk2~9yHu`rI+YbMwf+JLrc__Sw_`k|as`JKDW|D$)V<0T93~DYRj%OJVdGM{
zs&Cs*)5@7@oTkE&r9_X|u_$$4&8rlsQlLtKDg~+(s8XOxfhq;66sS_5N`Wc`sucMD
zOo6Wc{?5|wfrbMpUlbm_cKJM-5I+2lGkw=}ZRl<<?OTb_><`O9?MJkN)6@;r)RXtk
z6Rn<5fdj6!zmxwWZ?khgMuM~-CrUVFp5LzgeOd5U|Lu)$IA871Mb4AZ)ear+-p)s)
z<QQyFtZ?k@ZtU(KAX<sa93tU^IUEyEfjBihm2>=L@4N8_Nz_BG?m0H`^!~+P5j0mv
zel@gwU5r}8^FxyQ$KZ^qh)if7Bie;Kq%S{vkI2h^oTK@Dw=DnDjfNvP&e3ti=BaB|
z<(yckX`rGfbvN)1%skN4hcGAgb#`_3b@s$N6N0FG5|e3~P8O1ufq;}EZfQrHYWZ}s
zj3wz*1EQ<AlFHTTc{zHVFkPjtWUh|JtJ2a*cvY~nZh55<09%pXz7eR%n;L<-S0hmO
zY6R;3^XJ`nsTxeB+(|)xu4b-^+^JkqQo%hO9{>2A@7>Y0p~E2<m?vL}eCr!;<r?C3
z#TcnIq_9O%HB~84rNEz+0`B$CsN`<WU;S2YYH8R8=MAb<GMgF9<BcdYCq~R$POKr7
zw8|SpStEsAtdyZ)v%L-~+{mR9W_~aO#Uc_5LRQH{t>Ui|gbIYp|L0rcF+eXCT{D~l
z&S=R}x!c8BM%54a0=_1{zbO#a{lS*7za<dzgna(6FU*)Q^1QjDcGegQBhP|NLBMNT
zs_K}T&J1OYJa4OU;;OvWpsHHM+-|(mmCs|aMp|v(jbm4D$jpzaPPi3Y8>&@*(HzS(
z^=#hEtYO;Z4a>|HOVp&f(&j0a#{%#oF)|A$*G#;}!W}9DfuB!gGI>j3S7~#RO(ca~
z!AvFwGwBSE3H(iF6E1Gn35)?GUvThqt{9iOd6dLhz$x>SF`t>6AzzCx;PLsJWBzER
zT~8i@8pf$=-tSCK7+E7X%zK}lmg(oBYfh~cI*LMX*h)-IK&8`E{^BL)hW~coW9MG`
z0$==$Z@TMh9zVCV7ECjr&g7B>!xB(=2z=~JAyV2))YWFMsP-rLBi4V(#AmDTln*|H
zlQwcwdCVvz*|tk$Gr3VbV?5uBpTcvbP#Cvbyk6Av<V<g2#PHgtpf`){=uB?7X*@ZM
z^(;AuNnzv-EO0q$0d98%a16sUlpHk}s0~nG+aZlikVHV%8S=VDUEz>13}e@L*326o
zzVw*+VQ<<@6>&7P;Egu>L(y<RU)!I`;2sM!rH>Z}vze3-Nb58hx4h1IR$*~Y7<m9s
zh#3p~<iPo@05$Nk#2A3YaL%#!sljG)Rv|BX7ZAi?v14BXu_a?=a4^){og2z0^F~_l
z9z&mby)T*0m>?)F`-_=^p|9Q2-QCx}F7DB=MK2_2-bfbAe4;QpE<_r0MR!2vp7B;G
zuf$EA1A0$y-`2KmjzT3CM~%syn1?nq<RBk*z*!&P)X{%_ODC4?h%N}1GW2uww!)Zc
zjgNqn&KnRk56lHQ8};>_=(%MRMq%d7tT_xiXZ2yj9L^`lN5st##!!S1Q*}#F0VrjH
z!;F6EX>ZC%=2YvM0AZj58_;oo%T@pff;wi3p2}v%u+VUlp{2tpfSxEfayX@%w?=g^
z+S-A0yU7VXIS76wM+=zB&7oize3E)AZssO<$^aYCMpB7=9yJx>T}^&Zv{7%1cXZ$i
zFA~bho-(tn&(KaZ6Z~K1Xa@rI#QPg{96!e2wvM=(gDIQ64vq;3%;xtv0Q8vooyKq`
znbW7xx!dMH)}2h{Szs5GGxz#u8Tae$-~Ek-w={*?%cP(l3`ld!Eap=#1?ohF*C51z
zHJ;uj)*aRrx3j$QcvJhfrjW;{;cy;9#2m23F-f2z2F+qFZ6#7dQ}W=P`mS?x|N53t
zFc4AXw2EV6$^0Zvq)6ns0lU>|jGzYishEhhvRw3+OhMY96L1gQ&E%9R!Ohv*QO$eD
zjhw=f2`^V}!WzK=7^hBlsz|XevstWB%R7-Vc6x)MfMQ}vpHG;n<X|xi)X|%qJpo6*
zN_y?wxzn?A%xtof1y3e7WO_|2mCSo_YfS4w@@715^3Wh-6Ilc5N`SX}aN1QEnY6l+
zSxu0nme6zurly#Ze5g|qLCh1(s^lb=AeOgf<R`#Mm;sWQLSkB2SfFPySFnd92gShz
z1xHF84WR*MdAXG_mds>DuTBKVO{>r<HHhLsxruubFxU7Ti0Y_3vDssFcP7t4aSP{6
zJ^Z5S>Cijc2HNz^eZBF{{{HTs^-3F-u`1@7+zi_}Y&8ZcrPYLmx*A4BQM(4Y!ky=v
zB&%ELz*yz#NDO~wrQ<YHOG*cFKWijZmz6pdkjsDE>z`M|!#~>d2MvAmp*F{Ag&~p{
z%h<B)aQ2WIqsqCzGp}oHtpwN1CB~AwBt~xn?h`xWff?cc5%}u<6EC<&gOSg(Dr2*R
zIdTeqYFvZS)L^tZ7;1_}Q^}@KDA?Q-^9PKkp>V{P9tsW_QDaaoI(t3{uKw5oqeEED
zdVAhXrc*f8rL)1K!;&^K6R--B1-(4W^<FD$di2e5>Er3(q^@jpy(|!P%N#1~gi)av
zOjedz$RPL{QXSfE7;Y!{v?AVQ$>}9Aqgy+nIk~8WKv^21O!*uhUAzPXv08vowRC7(
zeN$6A^sNQaD@A>|VZtQ@nTSCL0S+Sq%V2CN8}WsB9tpl!W};w@XHp3s;IOGA($p@b
z4Zw!TWLZ~EmP`(#(a23?@@9^0#YTwLWDYV<o{$tP3fMBp76;aq*=F+q#H~P8Fk_>i
z%{b|m%C)NWne2fLa5cH4Mnf&A05yqh#VW9Is!NBSo!R<u$|}q$SRX|0^^e}^s9g<^
zOw#^O^u#gOpg;;4!@@HrLKHuRBAZVzaL)E$f;XX3=^=Rdswm`%czl<Pq=&+eLchoF
z^Z3yGc)>pS`ZzSpTAf88d<^I_Ko{LOz3ZO70c8eMvZ|a_*T?%6ty1O~^HV@``iHa4
zTsdrIfk}NO@tIL$xovXiq1}*!QXSn}y7@&jy{orR?``kz?Ay{d!1oti*tW5IV4L1n
zN$<E}!;&u9`%X)8z(|ecgi$TlW2!J+8A?}y+2<SwFD=_qP}Z&*y3}xT=+1egblfVz
zK-)J<VA`jP!q#P<sZbPLv>C}eBWk{M=k%>+epER!Xbx|S&0RwXSjLaT_Nuzm%L=vl
z!Y$zlG~EfRYa;<{q<?A$T>xcnFjmh3114rcTDlC2aZ&i-V%CPrArnYA#~qhrtfc-}
zt^P6U?|(1tZ&jAo#YG7B!X#U}1xyx!hBK_tW9oJlcCkbiK6d_Qe}DJ;743WQyA;3O
z_*wW}zj$T)C-D0?etYqo#P8M>E8B0v?|S?$$L~Y<eJc9%);m9Up!FO&(0ZKlU#%N|
z_;@SDe%^ZS4F_6pMcD|-{uO0kN7=(DYts(2zJ{{kgO9hmci$ie?L9XfV1HGxN`qry
z@4jK3{F2=S(%^eP!aN}@g?%hpcA6$%@+IwjjlRjGqAUJmx7RHc+1Yzp|Ja9>T*2F)
z@V+Mnb@#xcQ?sMLnA5wAL0v}_G1B7qxA;RkX0X3S&p|t}{%uO?YJKrkX&la)gUM`T
zvN6@@Sc*FQiGn5K9^O*KB#}#ba;YU#`tY>`A~W~lfa&bYVNT4sCtq`mKR9zwQ%5tm
zE(VH*-eIJ4f0zm8!@n?h;bVf0T6)ujj<BlUl-74O1{x!LG1(Yw4C}e^@g@<TY#JMc
z#nHrV{Qf}H9}M|?0mp6Fk3eoNB?!vY>&j>J9&<wHI4|T0m?StPIxKN0UiSAD)l4vG
zs++G%28~bhGRh+Kg%D2KYLa@;%P4vMes3V^JyL9!*usKsI|Uu~+4!W6I0eI1k!mRs
zW1dLFj#o&tLV2D%nK`i#TZdnHnj(^{_lQ6Zf-;*;DBva$Wrr)NZ$_{u2dzGm8CNR7
zF^w3W0C;3Bl`_|$CfM(Ed(}2Xr;?Gz$uw8m<%~-_=A(6KU?Ms$onxqBc8rt70vvdV
zM8=yF6T{h4&h$Wsh)0e;M}F?%r}j~V#%&TzaX{vWJ|OdLSJzJwTfka;@Pjm%FR!r)
zLmG=GehY{AAc6Q!6=~4d$4Fr{yLdYtqD$6014lp@2-TEC-XlD2(yqoS2q^-etf6;{
z7fx54d|``Qo+85W>Qr>#bKfixD<M&X&f3@nRgxBB@+H{~-rRdmd>KhQ86U2if|NrJ
z`|)X%qXlBmp~JG`=XY&1I=CZVlvmcHJ8n94Byvtm$zcB@VuYoj8C({`1sRfb9bn;B
ziW@2?T?I?0#=>~WKf_PWLJpd?D3@k9HI`OE6wk0V1R0*RlAd96!pnA2CgmL+v%D$<
zYDdwq-p1h*dW@9ED8l+qBID5dn9&I7rDA@S&{y5&5f-!%K=#1*%OMUf;^<a!P?dAM
zUf4}dqh=a*H`n!!V&%hlA2kYIAc>%8GTT&O3pkhKIx>jC+^VK`7zNzxA4_qXI4Z6G
zn$r4TDAzhx_cTjuE}{qlAMwp3(#kg2^_u)*3TcmN^j`7)?<Y=Mxl>;4P9Hi)UcGon
z%TsO(jboV4Jo1r8)mn9gtz?kdW*DP<n!{q-+`DnxdiY>#*G8H>KA#@+M4KCRzK-Z2
zPf%X_1GsLsucLa{6Y+^Ufq)+I1pM+kitAWVTn7Vs)DsHH>!=<BFmWC7<2n+P*I~Wc
z6O?VkK3q3r0|EUyuY-P_on`sJFK+w{DduSo@RNYNi3a&57#2^$A-)lv$56&MVa6)v
z@kjV362MIqz41+y`;U5}F>%8KjC#W2NsO_adL#M?Fgh4sGjNW`CLtb+Uji0m3^OiK
zPcQ;HG)qVr3mVBfuvkGWStsCU8cI08Fzksk-J*yR;My<SI<JFV?vr(Z1Nb7x5Q*VB
zDshWNaUJ10OnO|0nO`waAdKsvBw8ql>j2{emN0(>nPZ-qA9J8t^3k3T>q>irB@N(;
zW=Wf39>l*>17O>X6+vm&?0kEx&7~PU2Jft~4_#IZtPkLurHpa70QN<~O0o3!TD*6e
zG87(4ZFKkaZ|?5vL_`jiEYSn@VrJDMy=tRuy<tM73>(hIyrHW!g-e4}(i)Xj#S9um
z`R-~5C~u4-q=*=eFh-|abi;WSc%3S7-6E@OvXlkYT+9=kR=jDdp@Q)4lN^O3tz<`4
zG+BZqzqCYJ2yZ7#Vc3k1lX8fkwD;o^o`8<4n`)?mJxJQ~)9_c_Vv8-FFaG!s$5nDH
zUv!8m_LI(obE&hh*Am3>6yYZYU5=$P`yIx$a$nMojkEZai0azA@|AtesvpbS=d%!S
zN!5?7<aNJ!KX$$d&h^B5dD#e;Z98%$UhQvg$!$$+`ieb#1ff|ocq01R4LCf(M?>_p
z#nBK5vs}vbSVJB(Y}Dg0c*{@l@$He~pa+p+uEf7}=x1g>8V*$n4{;)EE`Br)oxn3?
z7aY-c9K$c2w&9Y)9jHH?W2w$-KR(LPrXS;1&u7YSeQn-}l?2!z$<NY}_L34{8a{N=
zz%q-Dx%|0<Zc(-xr7xfS?HenJvJqI6ih0vSSy)7sDvL5Dzc~DIq|}}!)u#1a6`~w>
zoLi*M9d@h+1OEbofxJEHKIyRrYgDa6!S)I#J@mCQ_9mniFZCId+X92Na+?egAtLt&
zY5A0`va*v{DM(bd!hqqylekAm;ONNhu^7JmD*{tm*&ge_$GBwK4t$_jT5Gq|5Iz{s
zy;Tr#+Q%U5SgO%o=OjPs9xR9`4}kKUKhuU?X|UK?SdKs*x+=%UW*_$K#^>xq#D5X-
z8IS0#B(clG(Y}Cz34>I74zWO;k>a|9vEb_Al?xw}S3A7cLyqHzvl&2Jv9}=%RCtem
zY|^#(nmJNEMXFB={wn#NZZu~zP?t;x@0$_4g2_7Lu#<eYF-_AKRZV3dIaMUCZd+Bi
zt+3~2Z?e64b6*~_t+dMoAz0E_<;eFY_o?D;`w#;TO84}7vE~o2yI9;+Q<Va9rhr@j
z&sFQsRoA~YCeE4o{|(@2Va1e<o$qS$tKp>e)zFuvu`|0((7AvbbcS-%HHU6<`*>?n
zYFxJXy_I~tqp+ze%?P4_k)qyK9M=6&9iBq485`}>)wN@}SBzNAZV^oBBG!UeJEj>N
zjxLnSVyoZ?1j%gn!8D@;@$QN!Ogl-gkU3(rClH&h2L+bLc%<)^9MM3l#@!qaPuM}9
zY8Yb5xu*(?vKoeX2cJor*|_sUohX$t7nxl4=BfrBE9kc&iBJA&!<q&Q1^-_HpHvbF
zksXny-Qz#2sPx_A1xs+s>Gb)c%}bZ?=Ue7!Dj<E94eR*j{63Z{Re4<}zh`|MUd2P;
zIng5ynTO(zw`ZN-#^A4uaO{GU%K4gSSsixd_$wy$45<9yeu%G_ti*GUh;Z17-+-$G
thheBMSa2lAdC%L0b*&7akB?&%>Xi)n;uuGp_+@EEY1S8|mo5B~@qai|1|t9f

literal 0
HcmV?d00001

diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 60c67568..7fb31c59 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -35,7 +35,7 @@ def test_get_local_mmd_git_path(dataDir):
 
 
 @pytest.mark.py_mmd_tools
-def test_mmd_change_file_location(dataDir):
+def test_mmd_change_file_location(dataDir, monkeypatch):
     """Test that an MMD file is created with new file_location, and
     new metadata update.
     """
@@ -64,9 +64,10 @@ def test_mmd_change_file_location(dataDir):
     os.remove(new_mmd)
 
     # Test that it fails when the mmd does not exist
-    mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False)
-    assert mmd is None
-    assert changed is False
+    with pytest.raises(ValueError):
+        mmd, changed = mmd_change_file_location(mmd, new_file_location, copy=False)
+        assert mmd is None
+        assert changed is False
 
 
 @pytest.mark.py_mmd_tools
@@ -88,21 +89,26 @@ def test_move_data(dataDir, monkeypatch):
     """Test the move_data function.
     """
     mmd_repository_path = "/some/folder/mmd-xml-production"
+    old_file_location_base = dataDir
     new_file_location_base = "/some/where/new"
     nc_file = os.path.join(dataDir, "reference_nc.nc")
 
     def mock_walk(*a, **k):
         yield (1, nc_file)
 
-    class MockResponse:
+    # This variable is needed when mocking datetime_glob.walk
+    pattern = os.path.join(dataDir, "%Y/%m/%d/*.nc")
 
+    class MockResponse:
         status_code = 200
 
+    # Test check for environment in move_data function
     with pytest.raises(ValueError):
         move_data(mmd_repository_path, new_file_location_base, nc_file, env="hei")
 
+    # Check that an error is raised if the given env is not in the MMD repo path
     with pytest.raises(ValueError):
-        move_data("/some/folder/mmd-xml-noenv", new_file_location_base, nc_file)
+        move_data("/some/folder/mmd-xml-noenv", new_file_location_base, nc_file, env="dev")
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
@@ -117,7 +123,8 @@ class MockResponse:
                    lambda *a, **k: MockResponse())
         mp.setattr("py_mmd_tools.mmd_operations.os.access",
                    lambda *a, **k: True)
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, nc_file)
+        # Successful call
+        not_updated, updated = move_data(mmd_repository_path, nc_file, new_file_location_base)
         assert len(not_updated) == 0
         assert len(updated) == 1
         assert os.path.isfile(updated[0])
@@ -126,34 +133,32 @@ class MockResponse:
             if "<mmd:file_location>" in line:
                 assert "/some/where/new" in line
 
-        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
-                   lambda *a, **k: mock_walk(*a, **k))
-        # This pattern is not really used but it should look something
-        # like this when used correctly:
-        pattern = os.path.join(dataDir, "%Y/%m/%d/*.nc")
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern,
-                                         dry_run=False)
-        assert len(not_updated) == 0
-        assert len(updated) == 1
-        assert os.path.isfile(updated[0])
-        lines = mmd_readlines(updated[0])
-        for line in lines:
-            if "<mmd:file_location>" in line:
-                assert "/some/where/new" in line
-        # TODO: Remove new MMD file - manual update needed for now..
-        # os.remove() - get filename through a function
+        # The ext_pattern input is tested in test_move_data_script
 
+        # Test that an exception is raised if we're not able to move the nc-file
+        mp.setattr("py_mmd_tools.mmd_operations.move_data_file", lambda *a, **k: (False, ""))
+        with pytest.raises(Exception):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern, dry_run=False)
+
+        # Test that an exception is raised if the MMD file cannot be
+        # updated with a new netCDF file location
         mp.setattr("py_mmd_tools.mmd_operations.mmd_change_file_location",
                    lambda *a, **k: (nc_file, False))
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
-        assert len(not_updated) == 1
+        with pytest.raises(Exception):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern)
 
         def raise_(ex):
             raise ex
 
+        # Test 
+        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
+                   lambda *a, **k: mock_walk(*a, **k))
         mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
                    lambda *a, **k: raise_(Exception("No path")))
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+        not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                         new_file_location_base, pattern)
         assert "Could not get MMD path" in not_updated[list(not_updated.keys())[0]]
 
     # Test os.access, remove_file_allowed is False
@@ -166,7 +171,23 @@ def raise_(ex):
         mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
                    lambda *a, **k: new_file_location_base)
         mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access)
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+        with pytest.raises(PermissionError):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern)
+
+    # Test os.access, write_file_allowed is False
+    mock_access = Mock()
+    mock_access.side_effect = [True, False]
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
+                   lambda *a, **k: mock_walk(*a, **k))
+        mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
+                   lambda *a, **k: new_file_location_base)
+        mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access)
+        with pytest.raises(PermissionError):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern)
 
     # Test os.access, remove_file_allowed and write_file_allowed are
     # False
@@ -178,7 +199,9 @@ def raise_(ex):
         mp.setattr("py_mmd_tools.mmd_operations.new_file_location",
                    lambda *a, **k: new_file_location_base)
         mp.setattr("py_mmd_tools.mmd_operations.os.access", mock_access)
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern)
+        with pytest.raises(PermissionError):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern)
 
     # Test second call to requests.post fails (dmci update)
     class MockResponseFail:
@@ -201,9 +224,9 @@ class MockResponseFail:
         mock_post = Mock()
         mock_post.side_effect = [MockResponse(), MockResponseFail()]
         mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern,
-                                         dry_run=False)
-        assert "Could not push updated" in not_updated[os.path.join(dataDir, "reference_nc.xml")]
+        with pytest.raises(Exception):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern, dry_run=False)
 
         # Check that updated is False if check_csw_catalog fails
         mock_post = Mock()
@@ -211,8 +234,8 @@ class MockResponseFail:
         mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
         mp.setattr("py_mmd_tools.mmd_operations.check_csw_catalog",
                    lambda *a, **k: (False, "Fail"))
-        not_updated, updated = move_data(mmd_repository_path, new_file_location_base, pattern,
-                                         dry_run=False)
+        not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                         new_file_location_base, pattern, dry_run=False)
         assert not_updated[os.path.join(dataDir, "reference_nc.xml")] == "Fail"
 
     subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
@@ -225,17 +248,16 @@ def test_new_file_location(monkeypatch):
     file = "/some/old/loc/2024/06/19/" \
            "S1A_IW_GRDM_1SDV_20240619T053156_20240619T053223_054388_069E03_FC95_MEPS.nc"
     new_base = "/some/where/else"
-    existing_pathname_pattern = "/some/old/loc/%Y/%m/%d/*.nc"
+    existing_base_loc = "/some/old/loc"
 
     with pytest.raises(ValueError):
-        new_file_location(file, new_base, existing_pathname_pattern)
+        new_file_location(file, new_base, existing_base_loc)
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", lambda *a, **k: True)
-        assert new_file_location(file, new_base, existing_pathname_pattern) == \
+        mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", lambda *a, **k: None)
+        assert new_file_location(file, new_base, existing_base_loc) == \
             "/some/where/else/2024/06/19"
-        new_base = "/some/where/else/2024/06/19"
-        assert new_file_location(file, new_base) == "/some/where/else/2024/06/19"
 
 
 @pytest.mark.py_mmd_tools
@@ -262,6 +284,17 @@ class MockResponse:
         assert found is False
         assert msg == "Could not find dataset in CSW catalog: /some/file.nc (id: no.met:123)"
 
+    class MockResponse:
+
+        status_code = 200
+
+    with monkeypatch.context() as mp:
+        mp.setattr("py_mmd_tools.mmd_operations.requests.get",
+                   lambda *a, **k: MockResponse())
+        found, msg = check_csw_catalog(ds_id, nc_file, urls, env)
+        assert found is True
+        assert msg == ""
+
 
 @pytest.mark.py_mmd_tools
 def test_move_data_file(monkeypatch):
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index 06e6694e..a9aa924b 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -20,13 +20,13 @@ def test_main(dataDir, monkeypatch):
     """
     """
     mmd_repository_path = "/some/folder/mmd-xml-production"
+    old_file_location_base = os.path.join(dataDir, "reference_nc.nc")
     new_file_location_base = "/some/where/new"
-    existing_pathname_pattern = os.path.join(dataDir, "reference_nc.nc")
     parser = create_parser()
     parsed = parser.parse_args([
         mmd_repository_path,
+        old_file_location_base,
         new_file_location_base,
-        existing_pathname_pattern
     ])
 
     class MockResponse:
@@ -40,13 +40,17 @@ class MockResponse:
                    lambda *a, **k: new_file_location_base)
         mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
                    lambda *a, **k: os.path.join(dataDir, "reference_nc.xml"))
-        mp.setattr("py_mmd_tools.mmd_operations.shutil.move",
-                   lambda *a, **k: None)
         mp.setattr("py_mmd_tools.mmd_operations.requests.get",
                    lambda *a, **k: MockResponse())
         mp.setattr("py_mmd_tools.mmd_operations.requests.post",
                    lambda *a, **k: MockResponse())
-        main(parsed)
+        mp.setattr("py_mmd_tools.mmd_operations.shutil.move", lambda *a, **k: None)
+        mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", lambda *a, **k: None)
+        mp.setattr("py_mmd_tools.mmd_operations.os.access",
+                   lambda *a, **k: True)
+        u, n = main(parsed)
+        assert len(u) == 1
+        subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
 
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",
                    lambda *a, **k: False)
@@ -55,7 +59,7 @@ class MockResponse:
 
         map = {
             mmd_repository_path: True,
-            new_file_location_base: False
+            new_file_location_base: False,
         }
 
         def mock_isdir(pp):
@@ -65,4 +69,95 @@ def mock_isdir(pp):
         with pytest.raises(ValueError):
             main(parsed)
 
-    subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+        map = {
+            mmd_repository_path: True,
+            new_file_location_base: True,
+        }
+        mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", mock_isdir)
+
+        old_file_location_base = dataDir
+
+        """ The following tests are basically testing datetime_glob.walk
+        but are included anyway...
+        """
+        # Day as format code
+        ext_pattern = "2024/09/%d/*.nc"
+        parser = create_parser()
+        parsed = parser.parse_args([
+            mmd_repository_path,
+            old_file_location_base,
+            new_file_location_base,
+            "--ext_pattern", ext_pattern,
+        ])
+        u, n = main(parsed)
+        assert len(u) == 1
+
+        subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+
+        # Month and day as format codes
+        ext_pattern = "2024/%m/%d/*.nc"
+        parser = create_parser()
+        parsed = parser.parse_args([
+            mmd_repository_path,
+            old_file_location_base,
+            new_file_location_base,
+            "--ext_pattern", ext_pattern,
+        ])
+        u, n = main(parsed)
+        assert len(u) == 1
+
+        subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+
+        # Year, month and day as format codes
+        ext_pattern = "%Y/%m/%d/*.nc"
+        parser = create_parser()
+        parsed = parser.parse_args([
+            mmd_repository_path,
+            old_file_location_base,
+            new_file_location_base,
+            "--ext_pattern", ext_pattern,
+        ])
+        u, n = main(parsed)
+        assert len(u) == 1
+
+        subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+
+        # Month as format code
+        ext_pattern = "2024/%m/01/*.nc"
+        parser = create_parser()
+        parsed = parser.parse_args([
+            mmd_repository_path,
+            old_file_location_base,
+            new_file_location_base,
+            "--ext_pattern", ext_pattern,
+        ])
+        u, n = main(parsed)
+        assert len(u) == 1
+
+        subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+
+        # Year as format code
+        ext_pattern = "%Y/09/01/*.nc"
+        parser = create_parser()
+        parsed = parser.parse_args([
+            mmd_repository_path,
+            old_file_location_base,
+            new_file_location_base,
+            "--ext_pattern", ext_pattern,
+        ])
+        u, n = main(parsed)
+        assert len(u) == 1
+
+        subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
+
+        # No match
+        ext_pattern = "%Y/09/02/*.nc"
+        parser = create_parser()
+        parsed = parser.parse_args([
+            mmd_repository_path,
+            old_file_location_base,
+            new_file_location_base,
+            "--ext_pattern", ext_pattern,
+        ])
+        u, n = main(parsed)
+        assert len(u) == 0

From ab4741203d023a1efc466114979ff98bc3897d7e Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 14:08:11 +0200
Subject: [PATCH 22/32] #337: raise exception and resolve flake errors

---
 py_mmd_tools/mmd_operations.py |  7 +------
 tests/test_mmd_operations.py   | 12 ++++++------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index c8abe0bc..dd67ddaf 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -8,7 +8,6 @@
 <https://github.com/metno/py-mmd-tools/blob/master/LICENSE>
 """
 import os
-import re
 import pytz
 import uuid
 import shutil
@@ -159,11 +158,7 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
         # Error message
         emsg = ""
         nfl = new_file_location(nc_file, new_file_location_base, old_file_location_base)
-        try:
-            mmd_orig = get_local_mmd_git_path(nc_file, mmd_repository_path)
-        except Exception as e:
-            not_updated[nc_file] = f"Could not get MMD path of {nc_file}.\nError: {str(e)}"
-            continue
+        mmd_orig = get_local_mmd_git_path(nc_file, mmd_repository_path)
 
         # Check permissions before doing anything
         remove_file_allowed = os.access(nc_file, os.W_OK)
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 7fb31c59..3dc147fc 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -152,14 +152,14 @@ class MockResponse:
         def raise_(ex):
             raise ex
 
-        # Test 
+        # Test that an error is raised if we cannot find the MMD path
         mp.setattr("py_mmd_tools.mmd_operations.datetime_glob.walk",
                    lambda *a, **k: mock_walk(*a, **k))
         mp.setattr("py_mmd_tools.mmd_operations.get_local_mmd_git_path",
                    lambda *a, **k: raise_(Exception("No path")))
-        not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
-                                         new_file_location_base, pattern)
-        assert "Could not get MMD path" in not_updated[list(not_updated.keys())[0]]
+        with pytest.raises(Exception):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern)
 
     # Test os.access, remove_file_allowed is False
     mock_access = Mock()
@@ -284,13 +284,13 @@ class MockResponse:
         assert found is False
         assert msg == "Could not find dataset in CSW catalog: /some/file.nc (id: no.met:123)"
 
-    class MockResponse:
+    class MockResponse2:
 
         status_code = 200
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.requests.get",
-                   lambda *a, **k: MockResponse())
+                   lambda *a, **k: MockResponse2())
         found, msg = check_csw_catalog(ds_id, nc_file, urls, env)
         assert found is True
         assert msg == ""

From 11ac8e1a1ada40cc03567a42781660f4af1bb296 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 14:41:22 +0200
Subject: [PATCH 23/32] #337: don't create folders in case of dry-run, change
 input arg and update README

---
 README.md                        | 7 ++++++-
 py_mmd_tools/mmd_operations.py   | 7 ++++---
 py_mmd_tools/script/move_data.py | 8 ++++----
 tests/test_mmd_operations.py     | 4 ++--
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 50fab451..bd8c3dbb 100644
--- a/README.md
+++ b/README.md
@@ -17,9 +17,14 @@ module can be extended with other necessary data management tools. Moving
 can be done with the `move_data` script, e.g.:
 
 ```
-move_data /path/to/files-from-git/mmd-xml-<env> /path/to/new/storage "/path/to/data/files/*.nc" --dmci-update
+move_data /path/to/files-from-git/mmd-xml-<env> /path/to/old/storage /path/to/new/storage "%Y/%m/%d/*.nc" --dmci-update
 ```
 
+The two last arguments provide a search pattern in case the netCDF files are
+stored in subfolders, and to directly updated the metadata catalog,
+respectively. If --dmci-update is not provided, local MMD files will not be
+pushed to the catalog.
+
 # Installation
 
 To avoid problems with conflicting versions, we recommend using the [Conda](
diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index dd67ddaf..8cca6f1b 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -157,7 +157,7 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
     for nc_file in existing:
         # Error message
         emsg = ""
-        nfl = new_file_location(nc_file, new_file_location_base, old_file_location_base)
+        nfl = new_file_location(nc_file, new_file_location_base, old_file_location_base, dry_run)
         mmd_orig = get_local_mmd_git_path(nc_file, mmd_repository_path)
 
         # Check permissions before doing anything
@@ -216,7 +216,7 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
     return not_updated, updated
 
 
-def new_file_location(nc_file, new_base_loc, existing_base_loc):
+def new_file_location(nc_file, new_base_loc, existing_base_loc, dry_run):
     """Return the name of the new folder where the netcdf file will be
     stored. Subfolders of new_base_loc will be created.
     """
@@ -224,5 +224,6 @@ def new_file_location(nc_file, new_base_loc, existing_base_loc):
         raise ValueError(f"Folder does not exist: {new_base_loc}")
     file_path = nc_file.replace(existing_base_loc, new_base_loc)
     new_folder = os.path.dirname(os.path.abspath(file_path))
-    os.makedirs(new_folder)
+    if not dry_run:
+        os.makedirs(new_folder)
     return new_folder
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index 81f470b1..0465dd5c 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -35,11 +35,11 @@ def create_parser():
         "new_file_location_base", type=str,
         help="Base or exact path to the folder to which the data file(s) will be moved.")
     parser.add_argument(
-        "--ext_pattern", type=str, default=None,
+        "--ext-pattern", type=str, default=None,
         help="Pathname pattern extending old_file_location_base, i.e., extending the "
-             "existing file *base* location(s). Allows parsing date/times from a path "
-             "given a glob pattern intertwined with date/time format akin to "
-             "strptime/strftime format.")
+             "existing file *base* location(s) with, e.g, the year and month as a "
+             "glob pattern intertwined with date/time format akin to "
+             "strptime/strftime format (e.g., '%Y/%m').")
     parser.add_argument(
         '--dmci-update', action='store_true',
         help='Directly update the online catalog with the changed MMD files.'
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 3dc147fc..c1452782 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -251,12 +251,12 @@ def test_new_file_location(monkeypatch):
     existing_base_loc = "/some/old/loc"
 
     with pytest.raises(ValueError):
-        new_file_location(file, new_base, existing_base_loc)
+        new_file_location(file, new_base, existing_base_loc, True)
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", lambda *a, **k: True)
         mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", lambda *a, **k: None)
-        assert new_file_location(file, new_base, existing_base_loc) == \
+        assert new_file_location(file, new_base, existing_base_loc, False) == \
             "/some/where/else/2024/06/19"
 
 

From cda8b899448f6cbf747f090d089d8768350ec3be Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 14:51:30 +0200
Subject: [PATCH 24/32] #337: change metadata update info

---
 py_mmd_tools/mmd_operations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 8cca6f1b..0aac623a 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -80,7 +80,7 @@ def mmd_change_file_location(mmd, new_file_location, copy=True):
     with open(mmd, "w") as f:
         for line in lines:
             if "</mmd:last_metadata_update>" in line:
-                add_metadata_update_info(f, "New file location in storage information.")
+                add_metadata_update_info(f, "New storage information.")
             if "<mmd:file_location>" in line:
                 f.write(f"    <mmd:file_location>{new_file_location}</mmd:file_location>\n")
                 status = True

From 236d8286cccb13a67c111f91481ae4a0b23a2bcb Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 14:53:49 +0200
Subject: [PATCH 25/32] #337: update test

---
 tests/test_move_data_script.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index a9aa924b..7003d71c 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -87,7 +87,7 @@ def mock_isdir(pp):
             mmd_repository_path,
             old_file_location_base,
             new_file_location_base,
-            "--ext_pattern", ext_pattern,
+            "--ext-pattern", ext_pattern,
         ])
         u, n = main(parsed)
         assert len(u) == 1
@@ -101,7 +101,7 @@ def mock_isdir(pp):
             mmd_repository_path,
             old_file_location_base,
             new_file_location_base,
-            "--ext_pattern", ext_pattern,
+            "--ext-pattern", ext_pattern,
         ])
         u, n = main(parsed)
         assert len(u) == 1
@@ -115,7 +115,7 @@ def mock_isdir(pp):
             mmd_repository_path,
             old_file_location_base,
             new_file_location_base,
-            "--ext_pattern", ext_pattern,
+            "--ext-pattern", ext_pattern,
         ])
         u, n = main(parsed)
         assert len(u) == 1
@@ -129,7 +129,7 @@ def mock_isdir(pp):
             mmd_repository_path,
             old_file_location_base,
             new_file_location_base,
-            "--ext_pattern", ext_pattern,
+            "--ext-pattern", ext_pattern,
         ])
         u, n = main(parsed)
         assert len(u) == 1
@@ -143,7 +143,7 @@ def mock_isdir(pp):
             mmd_repository_path,
             old_file_location_base,
             new_file_location_base,
-            "--ext_pattern", ext_pattern,
+            "--ext-pattern", ext_pattern,
         ])
         u, n = main(parsed)
         assert len(u) == 1
@@ -157,7 +157,7 @@ def mock_isdir(pp):
             mmd_repository_path,
             old_file_location_base,
             new_file_location_base,
-            "--ext_pattern", ext_pattern,
+            "--ext-pattern", ext_pattern,
         ])
         u, n = main(parsed)
         assert len(u) == 0

From e73d5b2484eb9cab5598089f54839df54ec04a9e Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 15:04:52 +0200
Subject: [PATCH 26/32] #337: change metadata update info

---
 py_mmd_tools/mmd_operations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 0aac623a..96a1d24a 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -80,7 +80,7 @@ def mmd_change_file_location(mmd, new_file_location, copy=True):
     with open(mmd, "w") as f:
         for line in lines:
             if "</mmd:last_metadata_update>" in line:
-                add_metadata_update_info(f, "New storage information.")
+                add_metadata_update_info(f, "Change storage information.")
             if "<mmd:file_location>" in line:
                 f.write(f"    <mmd:file_location>{new_file_location}</mmd:file_location>\n")
                 status = True

From 18732b645f57a9a3bde0fc429494a7fa12a0115a Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 15:58:05 +0200
Subject: [PATCH 27/32] #337: catch FileExistsError

---
 py_mmd_tools/mmd_operations.py | 6 +++++-
 tests/test_mmd_operations.py   | 8 ++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 96a1d24a..21663ec0 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -225,5 +225,9 @@ def new_file_location(nc_file, new_base_loc, existing_base_loc, dry_run):
     file_path = nc_file.replace(existing_base_loc, new_base_loc)
     new_folder = os.path.dirname(os.path.abspath(file_path))
     if not dry_run:
-        os.makedirs(new_folder)
+        try:
+            os.makedirs(new_folder)
+        except FileExistsError:
+            # Do nothing
+            pass
     return new_folder
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index c1452782..1ae8a90b 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -253,12 +253,20 @@ def test_new_file_location(monkeypatch):
     with pytest.raises(ValueError):
         new_file_location(file, new_base, existing_base_loc, True)
 
+    def raise_(ex):
+        raise ex
+
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir", lambda *a, **k: True)
         mp.setattr("py_mmd_tools.mmd_operations.os.makedirs", lambda *a, **k: None)
         assert new_file_location(file, new_base, existing_base_loc, False) == \
             "/some/where/else/2024/06/19"
 
+        mp.setattr("py_mmd_tools.mmd_operations.os.makedirs",
+                   lambda *a, **k: raise_(FileExistsError))
+        assert new_file_location(file, new_base, existing_base_loc, False) == \
+            "/some/where/else/2024/06/19"
+
 
 @pytest.mark.py_mmd_tools
 def test_check_csw_catalog(monkeypatch):

From bda2cba28f6a4993bdf70de23bc81335af55a743 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 17:40:14 +0200
Subject: [PATCH 28/32] #337: handle special chars in requests, check response
 text, add and modify tests

---
 py_mmd_tools/mmd_operations.py | 24 +++++++++++++++---------
 tests/test_mmd_operations.py   | 26 ++++++++++++++++++++++++++
 tests/test_move_data_script.py |  1 +
 3 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 21663ec0..2913e15a 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -18,6 +18,8 @@
 import warnings
 import datetime_glob
 
+import urllib.parse
+
 
 def add_metadata_update_info(f, note, type="Minor modification"):
     """ Add update information """
@@ -34,15 +36,19 @@ def check_csw_catalog(ds_id, nc_file, urls, env, emsg=""):
     """Search for the dataset with id 'ds_id' in the CSW metadata
     catalog.
     """
+    payload = {
+        "service": "CSW",
+        "version": "2.0.2",
+        "request": "GetRepositoryItem",
+        "id": ds_id}
+
+    payload_str = urllib.parse.urlencode(payload, safe=":")
+
     ds_found_and_accessible = False
     res = requests.get(url=f"https://{urls[env]['csw']}/csw",
-                       params={
-                           "service": "CSW",
-                           "version": "2.0.2",
-                           "request": "GetRepositoryItem",
-                           "id": ds_id})
+                       params=payload_str)
     # TODO: check the data_access urls
-    if res.status_code == 200:
+    if res.status_code == 200 and "ExceptionText" not in res.text:
         ds_found_and_accessible = True
     else:
         emsg += f"Could not find dataset in CSW catalog: {nc_file} (id: {ds_id})"
@@ -183,10 +189,10 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
 
         # Update with dmci update
         dmci_updated = False
-        if res.status_code == 200 and not dry_run:
+        if res.status_code == 200 and "OK" in res.text and not dry_run:
             # be careful with this...
             res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data)
-        if res.status_code == 200:
+        if res.status_code == 200 and "OK" in res.text:
             # This should be the case for a dry-run and a valid xml
             dmci_updated = True
         else:
@@ -199,7 +205,7 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
         elif dmci_updated and dry_run:
             nc_moved = True
 
-        ds_id = f"no.met.{urls[env]['id_namespace']}:{os.path.basename(mmd_orig).split('.')[0]}"
+        ds_id = f"{urls[env]['id_namespace']}:{os.path.basename(mmd_orig).split('.')[0]}"
         if not dry_run:
             ds_found_and_accessible, emsg = check_csw_catalog(ds_id, nc_file, urls, env, emsg=emsg)
         else:
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 1ae8a90b..3e1306a1 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -101,6 +101,7 @@ def mock_walk(*a, **k):
 
     class MockResponse:
         status_code = 200
+        text = "OK"
 
     # Test check for environment in move_data function
     with pytest.raises(ValueError):
@@ -295,6 +296,7 @@ class MockResponse:
     class MockResponse2:
 
         status_code = 200
+        text = ""
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.requests.get",
@@ -304,6 +306,30 @@ class MockResponse2:
         assert msg == ""
 
 
+@pytest.mark.online
+def test_check_dataset_in_met_csw_catalog():
+    """Check that a known dataset is found.
+    """
+    ds_id = "no.met:806070da-e9f3-4d03-ba1d-26b843961634"
+    # Leads to internal server error:
+    # ds_id = "no.met:aaaffc75-a42f-4bd8-a1f5-c8e8774fd948"
+    # url: 
+    # "https://data.csw.met.no/csw?service=CSW&version=2.0.2"
+    # "&request=GetRepositoryItem&id=no.met:aaaffc75-a42f-4bd8-a1f5-c8e8774fd948"
+    nc_file = "ncfile.nc"
+    urls = {"prod": {"dmci": "dmci.s-enda.k8s.met.no",
+                     "csw": "data.csw.met.no",
+                     "id_namespace": "no.met"}}
+    env = "prod"
+    found, msg = check_csw_catalog(ds_id, nc_file, urls, env)
+    assert found is True
+    assert msg == ""
+
+    ds_id = "rubbish"
+    found, msg = check_csw_catalog(ds_id, nc_file, urls, env)
+    assert found is False
+
+
 @pytest.mark.py_mmd_tools
 def test_move_data_file(monkeypatch):
     """Test move_data_file function can't move (working move is
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index 7003d71c..c5f1cabd 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -32,6 +32,7 @@ def test_main(dataDir, monkeypatch):
     class MockResponse:
 
         status_code = 200
+        text = "OK"
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.os.path.isdir",

From 71c4ee3d58b02941ccc67bc846cef707cd529e13 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 19:07:23 +0200
Subject: [PATCH 29/32] #337: remove commented lines

---
 tests/test_mmd_operations.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 3e1306a1..11adc36f 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -311,11 +311,6 @@ def test_check_dataset_in_met_csw_catalog():
     """Check that a known dataset is found.
     """
     ds_id = "no.met:806070da-e9f3-4d03-ba1d-26b843961634"
-    # Leads to internal server error:
-    # ds_id = "no.met:aaaffc75-a42f-4bd8-a1f5-c8e8774fd948"
-    # url: 
-    # "https://data.csw.met.no/csw?service=CSW&version=2.0.2"
-    # "&request=GetRepositoryItem&id=no.met:aaaffc75-a42f-4bd8-a1f5-c8e8774fd948"
     nc_file = "ncfile.nc"
     urls = {"prod": {"dmci": "dmci.s-enda.k8s.met.no",
                      "csw": "data.csw.met.no",

From da9fc220d3fed04028b2bed5146d4fc358c5ef9b Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Wed, 2 Oct 2024 22:16:31 +0200
Subject: [PATCH 30/32] #337: add logging

---
 py_mmd_tools/mmd_operations.py   |  5 +++--
 py_mmd_tools/script/move_data.py | 13 ++++++++-----
 pytest.ini                       |  4 ++++
 tests/test_mmd_operations.py     |  5 +++--
 tests/test_move_data_script.py   |  4 ++--
 5 files changed, 20 insertions(+), 11 deletions(-)
 create mode 100644 pytest.ini

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 2913e15a..0da475e4 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -11,11 +11,11 @@
 import pytz
 import uuid
 import shutil
+import logging
 import netCDF4
 import datetime
 import requests
 import tempfile
-import warnings
 import datetime_glob
 
 import urllib.parse
@@ -212,10 +212,11 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
             ds_found_and_accessible = True
 
         if not ds_found_and_accessible:
-            warnings.warn(f"Could not find data in CSW catalog: {ds_id}")
+            logging.warning(f"Could not find data in CSW catalog: {ds_id}")
 
         if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]):
             updated.append(mmd_orig)
+            logging.info(f"Updated {mmd_orig}.")
         else:
             not_updated[mmd_orig] = emsg
 
diff --git a/py_mmd_tools/script/move_data.py b/py_mmd_tools/script/move_data.py
index 0465dd5c..528ffbea 100755
--- a/py_mmd_tools/script/move_data.py
+++ b/py_mmd_tools/script/move_data.py
@@ -10,11 +10,9 @@
 
 py-mmd-tools is licensed under the Apache License 2.0
 <https://github.com/metno/py-mmd-tools/blob/master/LICENSE>
-
-Usage:
-    move_data [-h] -i INPUT -n OUTPUT_DIR
 """
 import os
+import logging
 import argparse
 
 from py_mmd_tools.mmd_operations import move_data
@@ -41,9 +39,12 @@ def create_parser():
              "glob pattern intertwined with date/time format akin to "
              "strptime/strftime format (e.g., '%Y/%m').")
     parser.add_argument(
-        '--dmci-update', action='store_true',
-        help='Directly update the online catalog with the changed MMD files.'
+        "--dmci-update", action="store_true",
+        help="Directly update the online catalog with the changed MMD files."
     )
+    parser.add_argument(
+        "--log-file", type=str, default="move_data.log",
+        help="Log filename")
 
     return parser
 
@@ -57,6 +58,8 @@ def main(args=None):
     if not os.path.isdir(args.new_file_location_base):
         raise ValueError(f"Invalid input: {args.new_file_location_base}")
 
+    logging.basicConfig(filename=args.log_file, level=logging.INFO)
+
     not_updated, updated = move_data(args.mmd_repository_path,
                                      args.old_file_location_base,
                                      args.new_file_location_base,
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 00000000..d90899d6
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    py_mmd_tools: Core functionality tests
+    online: Tests requiring web access
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 11adc36f..b2ec954a 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -85,7 +85,7 @@ def test_mmd_readlines(dataDir):
 
 
 @pytest.mark.py_mmd_tools
-def test_move_data(dataDir, monkeypatch):
+def test_move_data(dataDir, monkeypatch, caplog):
     """Test the move_data function.
     """
     mmd_repository_path = "/some/folder/mmd-xml-production"
@@ -237,6 +237,7 @@ class MockResponseFail:
                    lambda *a, **k: (False, "Fail"))
         not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
                                          new_file_location_base, pattern, dry_run=False)
+        assert "Could not find data in CSW catalog" in caplog.record_tuples[0][2]
         assert not_updated[os.path.join(dataDir, "reference_nc.xml")] == "Fail"
 
     subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
@@ -307,7 +308,7 @@ class MockResponse2:
 
 
 @pytest.mark.online
-def test_check_dataset_in_met_csw_catalog():
+def test_check_dataset_in_met_csw_catalog(caplog):
     """Check that a known dataset is found.
     """
     ds_id = "no.met:806070da-e9f3-4d03-ba1d-26b843961634"
diff --git a/tests/test_move_data_script.py b/tests/test_move_data_script.py
index c5f1cabd..1b452924 100644
--- a/tests/test_move_data_script.py
+++ b/tests/test_move_data_script.py
@@ -15,8 +15,8 @@
 from py_mmd_tools.script.move_data import create_parser
 
 
-@pytest.mark.script
-def test_main(dataDir, monkeypatch):
+@pytest.mark.py_mmd_tools
+def test_main(dataDir, monkeypatch, caplog):
     """
     """
     mmd_repository_path = "/some/folder/mmd-xml-production"

From cbe816e99097493fbd5409d38e92ad9e3eebee91 Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Thu, 3 Oct 2024 09:07:52 +0200
Subject: [PATCH 31/32] #337: change warning msg

---
 py_mmd_tools/mmd_operations.py | 4 ++--
 tests/test_mmd_operations.py   | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 0da475e4..0a09831d 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -51,7 +51,7 @@ def check_csw_catalog(ds_id, nc_file, urls, env, emsg=""):
     if res.status_code == 200 and "ExceptionText" not in res.text:
         ds_found_and_accessible = True
     else:
-        emsg += f"Could not find dataset in CSW catalog: {nc_file} (id: {ds_id})"
+        emsg += f"Could not find dataset ({ds_id}) in CSW catalog: {nc_file}, {res.text}"
 
     return ds_found_and_accessible, emsg
 
@@ -212,7 +212,7 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
             ds_found_and_accessible = True
 
         if not ds_found_and_accessible:
-            logging.warning(f"Could not find data in CSW catalog: {ds_id}")
+            logging.warning(emsg)
 
         if all([mmd_updated, dmci_updated, nc_moved, ds_found_and_accessible]):
             updated.append(mmd_orig)
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index b2ec954a..9a93183c 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -237,7 +237,7 @@ class MockResponseFail:
                    lambda *a, **k: (False, "Fail"))
         not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
                                          new_file_location_base, pattern, dry_run=False)
-        assert "Could not find data in CSW catalog" in caplog.record_tuples[0][2]
+        assert "Fail" in caplog.record_tuples[0][2]
         assert not_updated[os.path.join(dataDir, "reference_nc.xml")] == "Fail"
 
     subprocess.run(["git", "restore", "tests/data/reference_nc.xml"])
@@ -286,13 +286,14 @@ def test_check_csw_catalog(monkeypatch):
     class MockResponse:
 
         status_code = 400
+        text = "Fail"
 
     with monkeypatch.context() as mp:
         mp.setattr("py_mmd_tools.mmd_operations.requests.get",
                    lambda *a, **k: MockResponse())
         found, msg = check_csw_catalog(ds_id, nc_file, urls, env)
         assert found is False
-        assert msg == "Could not find dataset in CSW catalog: /some/file.nc (id: no.met:123)"
+        assert msg == "Could not find dataset (no.met:123) in CSW catalog: /some/file.nc, Fail"
 
     class MockResponse2:
 

From 2a18d58303446e713a9fc2cbca2939f17fbcaf5c Mon Sep 17 00:00:00 2001
From: "Morten W. Hansen" <mortenwh@met.no>
Date: Thu, 3 Oct 2024 11:30:46 +0200
Subject: [PATCH 32/32] #337: delete and insert instead of update

---
 README.md                      |  6 ++++++
 py_mmd_tools/mmd_operations.py | 18 +++++++++++++++---
 tests/test_mmd_operations.py   | 12 +++++++++++-
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index bd8c3dbb..9d45bae8 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,12 @@ stored in subfolders, and to directly updated the metadata catalog,
 respectively. If --dmci-update is not provided, local MMD files will not be
 pushed to the catalog.
 
+The results of the `move_data` script will be logged to a file, which by
+default is named `move_data.log`. You can change the filename through the
+option `--log-file`. Due to a bug in pycsw, the file may contain warnings about
+not found datasets. This can be handled by reingesting the MMD files. Keep the
+log file, and get help from a data manager to handle this.
+
 # Installation
 
 To avoid problems with conflicting versions, we recommend using the [Conda](
diff --git a/py_mmd_tools/mmd_operations.py b/py_mmd_tools/mmd_operations.py
index 0a09831d..f90a4811 100644
--- a/py_mmd_tools/mmd_operations.py
+++ b/py_mmd_tools/mmd_operations.py
@@ -190,13 +190,25 @@ def move_data(mmd_repository_path, old_file_location_base, new_file_location_bas
         # Update with dmci update
         dmci_updated = False
         if res.status_code == 200 and "OK" in res.text and not dry_run:
-            # be careful with this...
-            res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data)
+            ds = netCDF4.Dataset(nc_file)
+            ds_id = f"{ds.naming_authority}:{ds.id}".strip()
+            # Delete dataset
+            del_res = requests.post(url=f"https://{urls[env]['dmci']}/v1/delete/{ds_id}")
+            if del_res.status_code != 200 or "OK" not in del_res.text:
+                raise Exception(f"Not able to delete dataset ({ds_id}): {del_res.text}")
+            # Reingest dataset
+            res = requests.post(url=f"https://{urls[env]['dmci']}/v1/insert", data=data)
+            """NOTE: because of a bug in pycsw, the below updated is
+                     replaced by delete and insert above
+            """
+            # Update dataset
+            # res = requests.post(url=f"https://{urls[env]['dmci']}/v1/update", data=data)
         if res.status_code == 200 and "OK" in res.text:
             # This should be the case for a dry-run and a valid xml
             dmci_updated = True
         else:
-            raise Exception(f"Could not push updated MMD file to the DMCI API: {mmd_new}")
+            raise Exception("Could not push updated MMD file to the "
+                            f"DMCI API: {mmd_new}, {res.text}")
 
         if dmci_updated and not dry_run:
             nc_moved, emsg = move_data_file(nc_file, nfl)
diff --git a/tests/test_mmd_operations.py b/tests/test_mmd_operations.py
index 9a93183c..1040b9e9 100644
--- a/tests/test_mmd_operations.py
+++ b/tests/test_mmd_operations.py
@@ -222,6 +222,8 @@ class MockResponseFail:
                    lambda *a, **k: True)
         mp.setattr("py_mmd_tools.mmd_operations.requests.get",
                    lambda *a, **k: MockResponse())
+
+        # Check that an exception is raised if delete fails
         mock_post = Mock()
         mock_post.side_effect = [MockResponse(), MockResponseFail()]
         mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
@@ -229,9 +231,17 @@ class MockResponseFail:
             not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
                                              new_file_location_base, pattern, dry_run=False)
 
+        # Check that an exception is raised if insert fails
+        mock_post = Mock()
+        mock_post.side_effect = [MockResponse(), MockResponse(), MockResponseFail()]
+        mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
+        with pytest.raises(Exception):
+            not_updated, updated = move_data(mmd_repository_path, old_file_location_base,
+                                             new_file_location_base, pattern, dry_run=False)
+
         # Check that updated is False if check_csw_catalog fails
         mock_post = Mock()
-        mock_post.side_effect = [MockResponse(), MockResponse()]
+        mock_post.side_effect = [MockResponse(), MockResponse(), MockResponse()]
         mp.setattr("py_mmd_tools.mmd_operations.requests.post", mock_post)
         mp.setattr("py_mmd_tools.mmd_operations.check_csw_catalog",
                    lambda *a, **k: (False, "Fail"))