From c26465216d64ee7f3cdf5c243259fdd446bc95a3 Mon Sep 17 00:00:00 2001 From: aamir-s18 Date: Mon, 12 Jun 2023 20:19:13 +0200 Subject: [PATCH 1/2] Add show progess bar + overwrite doc --- landsatxplore/earthexplorer.py | 48 ++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/landsatxplore/earthexplorer.py b/landsatxplore/earthexplorer.py index c20315d..8106180 100644 --- a/landsatxplore/earthexplorer.py +++ b/landsatxplore/earthexplorer.py @@ -22,7 +22,7 @@ DATA_PRODUCTS = { # Level 1 datasets "landsat_tm_c2_l1": ["5e81f14f92acf9ef", "5e83d0a0f94d7d8d", "63231219fdd8c4e5"], - "landsat_etm_c2_l1":[ "5e83d0d0d2aaa488", "5e83d0d08fec8a66"], + "landsat_etm_c2_l1": ["5e83d0d0d2aaa488", "5e83d0d08fec8a66"], "landsat_ot_c2_l1": ["5e81f14ff4f9941c", "5e81f14f92acf9ef"], # Level 2 datasets "landsat_tm_c2_l2": ["5e83d11933473426", "5e83d11933473426", "632312ba6c0988ef"], @@ -30,15 +30,17 @@ "landsat_ot_c2_l2": ["5e83d14f30ea90a9", "5e83d14fec7cae84", "632210d4770592cf"] } + def _get_token(body): """Get `csrf_token`.""" csrf = re.findall(r'name="csrf" value="(.+?)"', body)[0] - + if not csrf: raise EarthExplorerError("EE: login failed (csrf token not found).") return csrf + class EarthExplorer(object): """Access Earth Explorer portal.""" @@ -62,7 +64,8 @@ def login(self, username, password): "password": password, "csrf": csrf, } - rsp = self.session.post(EE_LOGIN_URL, data=payload, allow_redirects=True) + rsp = self.session.post( + EE_LOGIN_URL, data=payload, allow_redirects=True) if not self.logged_in(): raise EarthExplorerError("EE: login failed.") @@ -70,9 +73,9 @@ def login(self, username, password): def logout(self): """Log out from Earth Explorer.""" self.session.get(EE_LOGOUT_URL) - + def _download( - self, url, output_dir, timeout, chunk_size=1024, skip=False, overwrite=False + self, url, output_dir, timeout, chunk_size=1024, skip=False, overwrite=False, show_progress=True ): """Download remote file given its URL.""" # Check availability of the requested product @@ -114,18 +117,24 @@ def _download( headers=headers, timeout=timeout, ) as r: - with tqdm( - total=filesize, - unit_scale=True, - unit="B", - unit_divisor=1024, - initial=downloaded_bytes - ) as pbar: + if show_progress: + with tqdm( + total=filesize, + unit_scale=True, + unit="B", + unit_divisor=1024, + initial=downloaded_bytes + ) as pbar: + with open(local_filename, file_mode) as f: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + pbar.update(chunk_size) + else: with open(local_filename, file_mode) as f: for chunk in r.iter_content(chunk_size=chunk_size): if chunk: f.write(chunk) - pbar.update(chunk_size) return local_filename except requests.exceptions.Timeout: @@ -140,7 +149,8 @@ def _get_fileinfo(self, download_url, timeout, output_dir): download_url, stream=True, allow_redirects=True, timeout=timeout ) as r: file_size = int(r.headers.get("Content-Length")) - local_filename = r.headers["Content-Disposition"].split("=")[-1] + local_filename = r.headers["Content-Disposition"].split( + "=")[-1] local_filename = local_filename.replace('"', "") local_filename = os.path.join(output_dir, local_filename) except requests.exceptions.Timeout: @@ -157,6 +167,7 @@ def download( timeout=300, skip=False, overwrite=False, + show_progress=True, ): """Download a Landsat scene. @@ -172,6 +183,10 @@ def download( Connection timeout in seconds. skip : bool, optional Skip download, only returns the remote filename. + overwrite : bool, optional + Overwrite existing files. + show_progress : bool, optional + Show progress bar for download. Returns ------- @@ -194,11 +209,12 @@ def download( data_product_id=dataset_id, entity_id=entity_id ) filename = self._download( - url, output_dir, timeout=timeout, skip=skip, overwrite=overwrite + url, output_dir, timeout=timeout, skip=skip, overwrite=overwrite, show_progress=show_progress ) except EarthExplorerError: if id_count+1 < id_num: - print('Download failed with dataset id {:d} of {:d}. Re-trying with the next one.'.format(id_count+1, id_num)) + print( + 'Download failed with dataset id {:d} of {:d}. Re-trying with the next one.'.format(id_count+1, id_num)) pass else: print('None of the archived ids succeeded! Update necessary!') From a4b1f7af69f5f4ea415a363262df07d456e0c7be Mon Sep 17 00:00:00 2001 From: aamir-s18 Date: Mon, 12 Jun 2023 20:30:24 +0200 Subject: [PATCH 2/2] add show-progress flag to CLI --- landsatxplore/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/landsatxplore/cli.py b/landsatxplore/cli.py index c6b8d49..fd5ca9e 100644 --- a/landsatxplore/cli.py +++ b/landsatxplore/cli.py @@ -152,6 +152,7 @@ def search( ) @click.option("--skip", is_flag=True, default=False) @click.option("--overwrite", is_flag=True, default=False) +@click.option("--show-progress", is_flag=True, default=True) @click.argument("scenes", type=click.STRING, nargs=-1) def download(username, password, dataset, output, timeout, skip, overwrite, scenes): """Download one or several scenes."""