Switch to more flexible tqmd progress bar

Author: tmorrell

caltechdata_api/download_file.py

@@ -1,5 +1,5 @@
 import requests, argparse
-from tqdm import tnrange, tqdm_notebook
+from tqdm.auto import tqdm
 
 
 def download_url(doi, media_type=None):
@@ -31,7 +31,7 @@ def download_file(doi, fname=None, media_type=None):
     else:
         with open(fname, "wb") as f:
             total_length = int(r.headers.get("content-length"))
-            pbar = tnrange(int(total_length / 1024), unit="B")
+            pbar = tqdm(total=int(total_length / 1024), unit="B")
             for chunk in r.iter_content(chunk_size=1024):
                 if chunk:
                     pbar.update()

example_download_and_upload.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -18,22 +18,22 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "7e6735a34e5c4448bbeb72920ed7b963",
+       "model_id": "f2aac6cbb1ef448f8899b204ffb1dc2c",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "HBox(children=(IntProgress(value=0, max=665896), HTML(value='')))"
+       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=15990.0), HTML(value='')))"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
-    "#By default will download to file named 10.22002-D1.1945\n",
+    "#By default will download to file named 10.22002-D1.1098\n",
     "#Can provide filename of interest using fname option\n",
-    "filen = download_file('10.22002/D1.1945')"
+    "filen = download_file('10.22002/D1.1098')"
    ]
   },
   {
@@ -88,9 +88,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.4"
+   "version": "3.8.5"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

tests/10.22002-D1.1098

@@ -21,16 +21,16 @@
 def load_json_path(path):
     """Helper method for loading a JSON example file from a path."""
     path_base = dirname(__file__)
-    with io.open(join(path_base, path), encoding='utf-8') as file:
+    with io.open(join(path_base, path), encoding="utf-8") as file:
         content = file.read()
     return json.loads(content)
 
+
 def write_json_path(path, metadata):
     """Helper method for writing a JSON example file to a path."""
     path_base = dirname(__file__)
     path_full = join(path_base, path)
     print(path_full)
     print(metadata)
-    with io.open(path_full, 'w', encoding='utf-8') as file:
+    with io.open(path_full, "w", encoding="utf-8") as file:
         json.dump(metadata, file)
-

tests/helpers.py

@@ -18,48 +18,47 @@
 from helpers import load_json_path, write_json_path
 
 CALTECHDATA_FILES = [
-    'data/caltechdata/210.json',
-    'data/caltechdata/266.json',
-    'data/caltechdata/267.json',
-    'data/caltechdata/268.json',
-    'data/caltechdata/283.json',
-    'data/caltechdata/293.json',
-    'data/caltechdata/301.json',
-    'data/caltechdata/970.json',
-    'data/caltechdata/1171.json',
-    'data/caltechdata/1235.json',
-    'data/caltechdata/1250.json',
-    'data/caltechdata/1259.json',
-    'data/caltechdata/1300.json',
+    "data/caltechdata/210.json",
+    "data/caltechdata/266.json",
+    "data/caltechdata/267.json",
+    "data/caltechdata/268.json",
+    "data/caltechdata/283.json",
+    "data/caltechdata/293.json",
+    "data/caltechdata/301.json",
+    "data/caltechdata/970.json",
+    "data/caltechdata/1171.json",
+    "data/caltechdata/1235.json",
+    "data/caltechdata/1250.json",
+    "data/caltechdata/1259.json",
+    "data/caltechdata/1300.json",
 ]
 
 DATACITE4_FILES = [
-    'data/datacite4/210.json',
-    'data/datacite4/266.json',
-    'data/datacite4/267.json',
-    'data/datacite4/268.json',
-    'data/datacite4/283.json',
-    'data/datacite4/293.json',
-    'data/datacite4/301.json',
-    'data/datacite4/970.json',
-    'data/datacite4/1171.json',
-    'data/datacite4/1235.json',
-    'data/datacite4/1250.json',
-    'data/datacite4/1259.json',
-    'data/datacite4/1300.json',
+    "data/datacite4/210.json",
+    "data/datacite4/266.json",
+    "data/datacite4/267.json",
+    "data/datacite4/268.json",
+    "data/datacite4/283.json",
+    "data/datacite4/293.json",
+    "data/datacite4/301.json",
+    "data/datacite4/970.json",
+    "data/datacite4/1171.json",
+    "data/datacite4/1235.json",
+    "data/datacite4/1250.json",
+    "data/datacite4/1259.json",
+    "data/datacite4/1300.json",
 ]
 
 
-@pytest.mark.parametrize('example_caltechdata', CALTECHDATA_FILES)
+@pytest.mark.parametrize("example_caltechdata", CALTECHDATA_FILES)
 def test_example_json_validates(example_caltechdata):
     """Test the example file validates against the JSON schema after
     decustomizing to DataCite."""
     example_json = load_json_path(example_caltechdata)
     datacite4 = decustomize_schema(example_json)
     validator4.validate(datacite4)
     example_json = load_json_path(example_caltechdata)
-    datacite43 = decustomize_schema(example_json,schema="43")
+    datacite43 = decustomize_schema(example_json, schema="43")
     validator43.validate(datacite43)
-    outname = example_caltechdata.split('/')[-1]
+    outname = example_caltechdata.split("/")[-1]
     write_json_path(f"data/datacite43/{outname}", datacite43)
-

tests/test_conversion.py

@@ -14,8 +14,13 @@
 
 from caltechdata_api import download_url, download_file
 
+
 def test_download():
     """Test that downloads from the DataCite Media API work."""
-    example_doi = '10.22002/D1.1945'
-    expected_url = 'https://data.caltech.edu/tindfiles/serve/0cedc067-1ff0-4fc9-871a-2869f3a4957d/'
+    example_doi = "10.22002/D1.1098"
+    expected_url = (
+        "https://data.caltech.edu/tindfiles/serve/293d37c5-73f2-4016-bcd5-76cf353ff9d8/"
+    )
     assert expected_url == download_url(example_doi)
+    filen = download_file(example_doi)
+    assert filen == "10.22002-D1.1098"