Skip to content

Commit

Permalink
Merge branch 'main' into fix-theme-furo
Browse files Browse the repository at this point in the history
  • Loading branch information
Thakor-Yashpal authored Nov 15, 2024
2 parents 9d96575 + ba0680a commit c73c4ac
Show file tree
Hide file tree
Showing 9 changed files with 70 additions and 74 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,4 @@ jobs:
name: pypi-dist
path: dist/
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@v1.11.0
uses: pypa/gh-action-pypi-publish@v1.12.2
4 changes: 2 additions & 2 deletions requirements/required.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# setup
setuptools==75.3.0
setuptools==75.5.0

# install
einops==0.8.0
Expand All @@ -18,5 +18,5 @@ segmentation-models-pytorch==0.3.4
shapely==2.0.6
timm==0.9.7
torch==2.5.1
torchmetrics==1.5.2
torchmetrics==1.6.0
torchvision==0.20.1
24 changes: 6 additions & 18 deletions tests/data/nlcd/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import hashlib
import os
import shutil

import numpy as np
import rasterio
Expand All @@ -16,8 +15,6 @@

np.random.seed(0)

dir = 'nlcd_{}_land_cover_l48_20210604'

years = [2011, 2019]

wkt = """
Expand Down Expand Up @@ -67,21 +64,12 @@ def create_file(path: str, dtype: str) -> None:

if __name__ == '__main__':
for year in years:
year_dir = dir.format(year)
# Remove old data
if os.path.isdir(year_dir):
shutil.rmtree(year_dir)

os.makedirs(os.path.join(os.getcwd(), year_dir))

zip_filename = year_dir + '.zip'
filename = year_dir + '.img'
create_file(os.path.join(year_dir, filename), dtype='int8')

# Compress data
shutil.make_archive(year_dir, 'zip', '.', year_dir)
filename = os.path.join(
'tests', 'data', 'nlcd', 'Annual_NLCD_LndCov_{}_CU_C1V0.tif'
).format(year)
create_file(filename, dtype='int8')

# Compute checksums
with open(zip_filename, 'rb') as f:
with open(filename, 'rb') as f:
md5 = hashlib.md5(f.read()).hexdigest()
print(f'{zip_filename}: {md5}')
print(f'{filename}: {md5}')
Binary file not shown.
Binary file not shown.
12 changes: 5 additions & 7 deletions tests/datasets/test_nlcd.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,12 @@ class TestNLCD:
@pytest.fixture
def dataset(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> NLCD:
md5s = {
2011: '99546a3b89a0dddbe4e28e661c79984e',
2019: 'a4008746f15720b8908ddd357a75fded',
2011: '3346297a3cb53c9bd1c7e03b2e6e2d74',
2019: 'a307cdaa1add9dae05efe02fec4c33bb',
}
monkeypatch.setattr(NLCD, 'md5s', md5s)

url = os.path.join(
'tests', 'data', 'nlcd', 'nlcd_{}_land_cover_l48_20210604.zip'
)
url = os.path.join('tests', 'data', 'nlcd', 'Annual_NLCD_LndCov_{}_CU_C1V0.tif')
monkeypatch.setattr(NLCD, 'url', url)
monkeypatch.setattr(plt, 'show', lambda *args: None)
root = tmp_path
Expand Down Expand Up @@ -75,7 +73,7 @@ def test_already_extracted(self, dataset: NLCD) -> None:

def test_already_downloaded(self, tmp_path: Path) -> None:
pathname = os.path.join(
'tests', 'data', 'nlcd', 'nlcd_2019_land_cover_l48_20210604.zip'
'tests', 'data', 'nlcd', 'Annual_NLCD_LndCov_2019_CU_C1V0.tif'
)
root = tmp_path
shutil.copy(pathname, root)
Expand All @@ -86,7 +84,7 @@ def test_invalid_year(self, tmp_path: Path) -> None:
AssertionError,
match='NLCD data product only exists for the following years:',
):
NLCD(tmp_path, years=[1996])
NLCD(tmp_path, years=[1984])

def test_invalid_classes(self) -> None:
with pytest.raises(AssertionError):
Expand Down
102 changes: 56 additions & 46 deletions torchgeo/datasets/nlcd.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

"""NLCD dataset."""

import glob
import os
from collections.abc import Callable, Iterable
from typing import Any, ClassVar
Expand All @@ -15,20 +14,18 @@

from .errors import DatasetNotFoundError
from .geo import RasterDataset
from .utils import BoundingBox, Path, download_url, extract_archive
from .utils import BoundingBox, Path, download_url


class NLCD(RasterDataset):
"""National Land Cover Database (NLCD) dataset.
"""Annual National Land Cover Database (NLCD) dataset.
The `NLCD dataset
<https://www.usgs.gov/centers/eros/science/national-land-cover-database>`_
is a land cover product that covers the United States and Puerto Rico. The current
implementation supports maps for the continental United States only. The product is
a joint effort between the United States Geological Survey
The `Annual NLCD products
<https://www.usgs.gov/centers/eros/science/annual-national-land-cover-database>`_
is an annual land cover product for the conterminous U.S. initially covering the period
from 1985 to 2023. The product is a joint effort between the United States Geological Survey
(`USGS <https://www.usgs.gov/>`_) and the Multi-Resolution Land Characteristics
Consortium (`MRLC <https://www.mrlc.gov/>`_) which released the first product
in 2001 with new updates every five years since then.
Consortium (`MRLC <https://www.mrlc.gov/>`_).
The dataset contains the following 17 classes:
Expand Down Expand Up @@ -57,33 +54,60 @@ class NLCD(RasterDataset):
* single channel .img file with integer class labels
If you use this dataset in your research, please use the corresponding citation:
If you use this dataset in your research, please cite the following paper:
* 2001: https://doi.org/10.5066/P9MZGHLF
* 2006: https://doi.org/10.5066/P9HBR9V3
* 2011: https://doi.org/10.5066/P97S2IID
* 2016: https://doi.org/10.5066/P96HHBIE
* 2019: https://doi.org/10.5066/P9KZCM54
* https://doi.org/10.5066/P94UXNTS
.. versionadded:: 0.5
"""

filename_glob = 'nlcd_*_land_cover_l48_*.img'
filename_regex = (
r'nlcd_(?P<date>\d{4})_land_cover_l48_(?P<publication_date>\d{8})\.img'
)
zipfile_glob = 'nlcd_*_land_cover_l48_*.zip'
filename_glob = 'Annual_NLCD_LndCov_*_CU_C1V0.tif'
filename_regex = r'Annual_NLCD_LndCov_(?P<date>\d{4})_CU_C1V0\.tif'
date_format = '%Y'
is_image = False

url = 'https://s3-us-west-2.amazonaws.com/mrlc/nlcd_{}_land_cover_l48_20210604.zip'
url = 'https://s3-us-west-2.amazonaws.com/mrlc/Annual_NLCD_LndCov_{}_CU_C1V0.tif'

md5s: ClassVar[dict[int, str]] = {
2001: '538166a4d783204764e3df3b221fc4cd',
2006: '67454e7874a00294adb9442374d0c309',
2011: 'ea524c835d173658eeb6fa3c8e6b917b',
2016: '452726f6e3bd3f70d8ca2476723d238a',
2019: '82851c3f8105763b01c83b4a9e6f3961',
1985: 'a2e1c5f0b34e9b15a63a9dc10e8d3ec2',
1986: 'da1d08ca51ac43abc14711c8d6139f1d',
1987: '2cb85e8f077c227605cd7bac62a72a75',
1988: 'b20fb987cc30926d2d125d045e02626d',
1989: 'dbe851cbea34d0a57c2a94eb745a1267',
1990: '1927e0e040b9ff513ff039749b64919b',
1991: 'eca73474843d6c58693eba62d70e507c',
1992: '8beda41ba79000f55a8e9358ba3fa5a4',
1993: '1a023552967cdac1111e9968ea62c879',
1994: 'acc30ce4f6cdd78af5f7887d17ac4de3',
1995: 'f728e8fc231b2e8e74a14201f500543a',
1996: 'd2580904244f89b20d6258150fbf4161',
1997: 'fec4e08032e162f2cc7dbe019d042609',
1998: '87ea19434de96ea99cd5d7991042816c',
1999: 'd4133737f20e75f3bd3a5baa32a668da',
2000: 'e20b61bb2e7f4034a33c9fd536798a01',
2001: 'b1f46ace9aedd17a89efab489cb67bc3',
2002: '57bf60d7cd473096af3bb125391bde63',
2003: '5e346854da9abf739152e85fee4c7aff',
2004: '13136f271f53a454358eb7ec12bda686',
2005: 'f00b66b57a23eb49a077e88704964a91',
2006: '074ba90de5e62a37a5f001b7572f6baa',
2007: 'cdef29a191cf165baaae80857ce5a980',
2008: 'da907c76a1f12739333148504fd111c9',
2009: '47890b306b875e681990b3db0c709da3',
2010: '9a81f405f9e2f45d581078afd53c2d4b',
2011: '13f4ef40b204aa1108dc0599d9546701',
2012: '66b33146f9a9d9491be10c59c51e3e33',
2013: 'f8d230f7dea493c47fbc74984ff856cc',
2014: '68eb07ce86c1f7c2546ec43c2f9f7029',
2015: 'f5a1b59fe54a70752f544c06cb965be4',
2016: 'f0c2e74824fc281a57821e28e2c7fe6e',
2017: 'a0aa8be0ed7d637f0f88f26d3742b20e',
2018: 'a01f31547837ff1dfec1aba07b89bbec',
2019: 'fa738201cddc1393dac4383b6ce2561a',
2020: 'aa8f51690c7b01f3b3b413be9a7c36d6',
2021: '47fc1794a64704a918b6ad586df4267c',
2022: '11359748229e138cde971947864104a4',
2023: '498ff8a512d32fe905720796fdb7fd52',
}

cmap: ClassVar[dict[int, tuple[int, int, int, int]]] = {
Expand Down Expand Up @@ -111,7 +135,7 @@ def __init__(
paths: Path | Iterable[Path] = 'data',
crs: CRS | None = None,
res: float | None = None,
years: list[int] = [2019],
years: list[int] = [2023],
classes: list[int] = list(cmap.keys()),
transforms: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
cache: bool = True,
Expand Down Expand Up @@ -183,19 +207,14 @@ def __getitem__(self, query: BoundingBox) -> dict[str, Any]:

def _verify(self) -> None:
"""Verify the integrity of the dataset."""
# Check if the extracted files already exist
if self.files:
return

# Check if the zip files have already been downloaded
# Check if the TIFF files for the specified years have already been downloaded
exists = []
for year in self.years:
zipfile_year = self.zipfile_glob.replace('*', str(year), 1)
filename_year = self.filename_glob.replace('*', str(year), 1)
assert isinstance(self.paths, str | os.PathLike)
pathname = os.path.join(self.paths, '**', zipfile_year)
if glob.glob(pathname, recursive=True):
pathname = os.path.join(self.paths, filename_year)
if os.path.exists(pathname):
exists.append(True)
self._extract()
else:
exists.append(False)

Expand All @@ -208,7 +227,6 @@ def _verify(self) -> None:

# Download the dataset
self._download()
self._extract()

def _download(self) -> None:
"""Download the dataset."""
Expand All @@ -219,14 +237,6 @@ def _download(self) -> None:
md5=self.md5s[year] if self.checksum else None,
)

def _extract(self) -> None:
"""Extract the dataset."""
for year in self.years:
zipfile_name = self.zipfile_glob.replace('*', str(year), 1)
assert isinstance(self.paths, str | os.PathLike)
pathname = os.path.join(self.paths, '**', zipfile_name)
extract_archive(glob.glob(pathname, recursive=True)[0], self.paths)

def plot(
self,
sample: dict[str, Any],
Expand Down

0 comments on commit c73c4ac

Please sign in to comment.