Some fixes to recent changes to support Python 2.7. Yes it is long si…

…nce EOL but there are still some distros that support it. The unit tests do pass on 2.7 even though they are not run in GH Actions. Updated unit tests to reflect 2.7 compatibility issues. Update Pipfile, Pipfile.lock, and setup.py.
fair-research · Mar 13, 2024 · 4c2d158 · 4c2d158
1 parent 13f1d2c
commit 4c2d158
Show file tree

Hide file tree

Showing 13 changed files with 525 additions and 339 deletions.
diff --git a/Pipfile b/Pipfile
@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-bdbag = {path = ".", extras = ["boto","globus"]}
+bdbag = {path = ".", extras = ["boto","globus","gcs"]}
 
 [dev-packages]
 mock = "*"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/bdbag/__init__.py b/bdbag/__init__.py
@@ -94,7 +94,7 @@ def stob(val):
     elif val in ('n', 'no', 'f', 'false', 'off', '0'):
         return False
     else:
-        raise ValueError(f"invalid truth value {val!r}")
+        raise ValueError("invalid truth value %r" % val)
 
 
 def get_typed_exception(e):

diff --git a/bdbag/bdbag_api.py b/bdbag/bdbag_api.py
@@ -363,14 +363,15 @@ def archive_bag(bag_path, bag_archiver, config_file=None, idempotent=None):
         tarmode = 'w:gz'
     elif bag_archiver == 'bz2':
         tarmode = 'w:bz2'
-    elif bag_archiver == 'xz':
+    elif bag_archiver == 'xz' and sys.version_info >= (3, 3):
         tarmode = 'w:xz'
     elif bag_archiver == 'zip':
         zfp = os.path.join(os.path.dirname(bag_path), fn)
         archive = zip_bag_dir(bag_path, zfp, idempotent)
     else:
         raise RuntimeError("Archive format not supported for bag file: %s \n "
-                           "Supported archive formats are ZIP or TAR/GZ/BZ2/XZ" % bag_path)
+                           "Supported archive formats are ZIP or TAR/GZ/BZ2%s" %
+                           (bag_path,  ("/XZ" if sys.version_info >= (3, 3) else "")))
 
     if tarmode:
         archive = tar_bag_dir(bag_path, fn, tarmode, idempotent)
@@ -432,34 +433,36 @@ def zip_bag_dir(bag_path, zip_file_path, idempotent=False):
     entries.sort()
     for e in entries:
         filepath = os.path.join(os.path.dirname(bag_path), e)
-        payload_dir = os.path.join(os.path.basename(bag_path), "data", "")
-        # a fixed mtime is a core requirement for a reproducible archive
-        if idempotent:
-            date_time = (1980, 1, 1, 0, 0, 0)
-        else:
-            st = os.stat(filepath)
-            mtime = time.localtime(st.st_mtime)
-            date_time = mtime[0:6]
-        info = ZipInfo(
-            filename=e,
-            date_time=date_time
-        )
-        info.create_system = 3  # unix
-        if e.endswith(os.path.sep):
-            info.external_attr = 0o40755 << 16 | 0x010
-            info.compress_type = ZIP_STORED
-            info.CRC = 0  # unclear why necessary, maybe a bug?
-            zipfile.writestr(info, b'')
+        if sys.version_info < (3,):
+            zipfile.write(filepath, e)
         else:
-            info.external_attr = 0o100644 << 16
-            info.compress_type = ZIP_DEFLATED
-            with io.open(filepath, 'rb') as data, zipfile.open(info, 'w') as out:
-                while True:
-                    chunk = data.read(io.DEFAULT_BUFFER_SIZE)
-                    if not chunk:
-                        break
-                    out.write(chunk)
-                out.flush()
+            if idempotent:
+                # a fixed mtime is a core requirement for a reproducible archive
+                date_time = (1980, 1, 1, 0, 0, 0)
+            else:
+                st = os.stat(filepath)
+                mtime = time.localtime(st.st_mtime)
+                date_time = mtime[0:6]
+            info = ZipInfo(
+                filename=e,
+                date_time=date_time
+            )
+            info.create_system = 3  # unix
+            if e.endswith(os.path.sep):
+                info.external_attr = 0o40755 << 16 | 0x010
+                info.compress_type = ZIP_STORED
+                info.CRC = 0  # unclear why necessary, maybe a bug?
+                zipfile.writestr(info, b'')
+            else:
+                info.external_attr = 0o100644 << 16
+                info.compress_type = ZIP_DEFLATED
+                with io.open(filepath, 'rb') as data, zipfile.open(info, 'w') as out:
+                    while True:
+                        chunk = data.read(io.DEFAULT_BUFFER_SIZE)
+                        if not chunk:
+                            break
+                        out.write(chunk)
+                    out.flush()
     zipfile.close()
     return zipfile.filename
 
@@ -489,12 +492,14 @@ def extract_bag(bag_path, output_path=None, temp=False, config_file=None):
             archive = ZipFile(bag_path)
             files = archive.namelist()
         elif tarfile.is_tarfile(bag_path):
-            logger.info("Extracting TAR/GZ/BZ2 archived file: %s" % bag_path)
+            logger.info("Extracting TAR/GZ/BZ2%s archived file: %s" %
+                        (bag_path,  ("/XZ" if sys.version_info >= (3, 3) else "")))
             archive = tarfile.open(bag_path)
             files = archive.getnames()
         else:
-            raise RuntimeError("Archive format not supported for file: %s"
-                               "\nSupported archive formats are ZIP or TAR/GZ/BZ2/XZ" % bag_path)
+            raise RuntimeError("Archive format not supported for file: %s\n"
+                               "Supported archive formats are ZIP or TAR/GZ/BZ2%s" %
+                               (bag_path,  ("/XZ" if sys.version_info >= (3, 3) else "")))
         archived_bag_dir = bag_parent_dir_from_archive(files)
         extracted_path = os.path.join(base_path, archived_bag_dir or bag_dir)
         output_path = os.path.join(output_path, extracted_path or bag_dir) if output_path else None

diff --git a/bdbag/bdbag_cli.py b/bdbag/bdbag_cli.py
@@ -90,9 +90,11 @@ def parse_cli():
              "directory will be deleted.")
 
     archiver_arg = "--archiver"
+    choices = ['zip', 'tar', 'tgz', 'bz2']
+    if sys.version_info >= (3, 3):
+        choices.append("xz")
     standard_args.add_argument(
-        archiver_arg, choices=['zip', 'tar', 'tgz', 'bz2', 'xz'],
-        help="Archive a bag using the specified format.")
+        archiver_arg, choices=choices, help="Archive a bag using the specified format.")
 
     idempotent_arg = "--idempotent"
     standard_args.add_argument(

diff --git a/setup.py b/setup.py
@@ -59,19 +59,19 @@
         'bagit_profile'
     ],
     install_requires=['pytz',
-                      'tzlocal<3; python_version < "3"',
+                      'tzlocal<3; python_version<"3"',
                       'tzlocal',
                       'certifi',
                       'packaging',
                       'importlib_metadata;python_version<"3.8"',
-                      'requests>=2.7.0',
+                      'requests',
                       'setuptools_scm<6.0',  # for bagit which does not properly include it in install_requires
                       'bagit==1.8.1',
                       'bagit-profile==1.3.1'
                       ],
     extras_require={
         'boto': ["boto3>=1.9.5", "botocore", "awscli"],
-        'globus': ["globus_sdk>=1.6.0"],
+        'globus': ["globus_sdk>=2,<4"],
         'gcs': ["google_cloud_storage"]
     },
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, <4',
@@ -84,6 +84,7 @@
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: Microsoft :: Windows",
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',

diff --git a/test/test-data/test-bag-profile/bag-info.txt b/test/test-data/test-bag-profile/bag-info.txt
@@ -1,5 +1,5 @@
 Bag-Software-Agent: BDBag version: 1.2.2 (Bagit version: 1.6.4) <https://github.com/fair-research/bdbag>
-BagIt-Profile-Identifier: file:./profiles/bdbag-profile.json
+BagIt-Profile-Identifier: file:profiles/bdbag-profile.json
 Bagging-Date: 2018-03-19
 Bagging-Time: 16:59:32 PST
 Contact-Name: mdarcy

diff --git a/test/test-data/test-bag-profile/tagmanifest-md5.txt b/test/test-data/test-bag-profile/tagmanifest-md5.txt
@@ -1,4 +1,4 @@
-92308824770d0db5049d2d3f3ac65fd6 bag-info.txt
+4945c64e4d6a4f27e6f35fdbace1aba8 bag-info.txt
 9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
 d0c7f2742808e0f0007f197339110959 manifest-md5.txt
 beb3e836fcab267224d2fc8d87e667db manifest-sha1.txt

diff --git a/test/test-data/test-bag-profile/tagmanifest-sha1.txt b/test/test-data/test-bag-profile/tagmanifest-sha1.txt
@@ -1,4 +1,4 @@
-b0c31f05cebab9f5f4df8e60b0fae2c0ba3d206a bag-info.txt
+f23beab33cb30b250c9fc1503814cd1773185f90 bag-info.txt
 e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt
 00f00590ea8c78c27fd5866f337fd900eeeb9d17 manifest-md5.txt
 c392c06018b4dcb5a20db90cb0e33164bd3e2bd6 manifest-sha1.txt

diff --git a/test/test-data/test-bag-profile/tagmanifest-sha256.txt b/test/test-data/test-bag-profile/tagmanifest-sha256.txt
@@ -1,4 +1,4 @@
-4a28828d28736c31796b78709ecc12fa7f1b9d16b2416729617a42d589ed69d1 bag-info.txt
+599b32d9d18db67f5295fb0c805bc12d638ddb88d1c336e5150f119f1308b9d3 bag-info.txt
 e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
 3bc03d737c6e0c7432ca7a5ac06c3f411292ff8ddd884c52f6b1df1abba50763 manifest-md5.txt
 70615718807d67bb652bfd1a3bdc35a7f99f8eda7458b073f5808db3f5c3b365 manifest-sha1.txt

diff --git a/test/test-data/test-bag-profile/tagmanifest-sha512.txt b/test/test-data/test-bag-profile/tagmanifest-sha512.txt
@@ -1,4 +1,4 @@
-f2f6c1516c2c20c502cffa9d884c4b2a4f9be37cb528563883e90400a2896d3e170109d837a8f8b5003e1a43da5d5832b395adc7628ab2874e539e3dcf9fd2ee bag-info.txt
+b42f94a1e7081b79c246cffce706d26fbf8282e64162b4a7f9bf14571d90a1f16c94aa3e61e70f381582b54263f20ff8b64a0df07d77269de07349f58f745c7a bag-info.txt
 418dcfbe17d5f4b454b18630be795462cf7da4ceb6313afa49451aa2568e41f7ca3d34cf0280c7d056dc5681a70c37586aa1755620520b9198eede905ba2d0f6 bagit.txt
 0eecbffb32b44584197afe7e983ebb8c3c321167e916a64f23c3ec8170c4940395da9820bb2074defadd77d9aa59c6aa890c1d652888c8ca12e46f403c540c98 manifest-md5.txt
 7444259042079d10f4ea436d598a7f3485d24bf7e790ff20c775213f22558cb08d92cae1bc7aa15dd83e8bb00873fae2963d8e6204adf172a8b1c147204f89b5 manifest-sha1.txt

diff --git a/test/test_api.py b/test/test_api.py
@@ -594,6 +594,7 @@ def test_archive_bag_bz2(self):
         except Exception as e:
             self.fail(get_typed_exception(e))
 
+    @unittest.skipIf(sys.version_info < (3, 3), 'Python version not supported')
     def test_archive_bag_xz(self):
         logger.info(self.getTestHeader('archive bag bz2 format'))
         try:
@@ -631,6 +632,7 @@ def test_archive_bag_idempotent_tgz(self):
     def test_archive_bag_idempotent_bz2(self):
         self._test_archive_bag_idempotent("bz2")
 
+    @unittest.skipIf(sys.version_info < (3, 3), 'Python version not supported')
     def test_archive_bag_idempotent_xz(self):
         self._test_archive_bag_idempotent("xz")
 
@@ -768,6 +770,7 @@ def test_extract_bag_archive_tgz(self):
     def test_extract_bag_archive_bz2(self):
         self._test_extract_bag_archive_tar("bz2")
 
+    @unittest.skipIf(sys.version_info < (3, 3), 'Python version not supported')
     def test_extract_bag_archive_xz(self):
         self._test_extract_bag_archive_tar("xz")
 

diff --git a/test/test_cli.py b/test/test_cli.py
@@ -126,6 +126,7 @@ def test_archive_tgz(self):
     def test_archive_bz2(self):
         self._test_archive("bz2")
 
+    @unittest.skipIf(sys.version_info < (3, 3), 'Python version not supported')
     def test_archive_xz(self):
         self._test_archive("xz")
 
@@ -141,6 +142,7 @@ def test_archive_idempotent_tgz(self):
     def test_archive_idempotent_bz2(self):
         self._test_archive("bz2", True)
 
+    @unittest.skipIf(sys.version_info < (3, 3), 'Python version not supported')
     def test_archive_idempotent_xz(self):
         self._test_archive("xz", True)