Skip to content

Commit

Permalink
Merge pull request #11 from bit-bots/update_storage_location
Browse files Browse the repository at this point in the history 
Update storage location
  • Loading branch information
@Flova
Flova authored Nov 18, 2022
2 parents d2b48e6 + cea2397 commit 9325d91
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 68 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ With annotations, segmentation mask and depth image

### Manual Download

The images and annotations can be downloaded here: https://cloud.crossmodal-learning.org/s/3wt3Sgyxc7pC5QT
The images and annotations can be downloaded here: https://data.bit-bots.de/TORSO-21/

### Automated Download

Expand Down
97 changes: 30 additions & 67 deletions scripts/download_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,25 @@
from urllib.request import urlretrieve

DATA_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), '../data'))
DOWNLOAD_LINK = 'https://cloud.crossmodal-learning.org/s/3wt3Sgyxc7pC5QT'
DOWNLOAD_LINK = 'https://data.bit-bots.de/TORSO-21'


def download(filename, params, folder, approx_size):
def download_and_extract_zip(filename, file_url, folder, approx_size):
print(f'Downloading dataset to {folder}... '
f'This might take a lot of time and take up to {approx_size} GB of disk space')
os.makedirs(folder, exist_ok=True)
query = urlencode(params)
urlretrieve(DOWNLOAD_LINK + '/download?' + query, filename)
urlretrieve(f'{DOWNLOAD_LINK}/{file_url}', filename)
print('Download finished, extracting data...')
with zipfile.ZipFile(filename) as f:
f.extractall(folder)
os.remove(filename)
print('Extraction finished.')


def download_file(filename, params):
def download_file(filename, file_url):
print(f'Downloading annotations file to {filename}...')
os.makedirs(os.path.dirname(filename), exist_ok=True)
query = urlencode(params)
urlretrieve(DOWNLOAD_LINK + '/download?' + query, filename)
urlretrieve(f'{DOWNLOAD_LINK}/{file_url}', filename)
print('Download finished.')


Expand All @@ -50,93 +48,58 @@ def download_file(filename, params):
parser.error('--all cannot be used with other options.')

if args.all:
tmp_file = os.path.join(DATA_FOLDER, 'dataset.zip')
params = {}
folder = DATA_FOLDER
approx_size = 190
download(tmp_file, params, folder, approx_size)
tmp_file = os.path.join(DATA_FOLDER, 'reality.zip')
approx_size = 11
download_and_extract_zip(tmp_file, 'reality.zip', DATA_FOLDER, approx_size)
tmp_file = os.path.join(DATA_FOLDER, 'simulation.zip')
approx_size = 180
download_and_extract_zip(tmp_file, 'simulation.zip', DATA_FOLDER, approx_size)
elif args.real and args.test and args.annotations:
download_file(os.path.join(DATA_FOLDER, 'reality', 'test', 'annotations.yaml'),
{'path': '/reality/test', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'reality', 'test', 'annotations.yaml'), 'reality/test/annotations.yaml')
elif args.real and args.test:
tmp_file = os.path.join(DATA_FOLDER, 'reality', 'test.zip')
params = {
'path': '/reality',
'files': 'test',
}
folder = os.path.join(DATA_FOLDER, 'reality')
approx_size = 2
download(tmp_file, params, folder, approx_size)
download_and_extract_zip(tmp_file, 'reality/test.zip', folder, approx_size)
elif args.real and args.train and args.annotations:
download_file(os.path.join(DATA_FOLDER, 'reality', 'train', 'annotations.yaml'),
{'path': '/reality/train', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'reality', 'train', 'annotations.yaml'), 'reality/train/annotations.yaml')
elif args.real and args.train:
tmp_file = os.path.join(DATA_FOLDER, 'reality', 'train.zip')
params = {
'path': '/reality',
'files': 'train',
}
folder = os.path.join(DATA_FOLDER, 'reality')
approx_size = 10
download(tmp_file, params, folder, approx_size)
download_and_extract_zip(tmp_file, 'reality/train.zip', folder, approx_size)
elif args.real and args.annotations:
download_file(os.path.join(DATA_FOLDER, 'reality', 'train', 'annotations.yaml'),
{'path': '/reality/train', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'reality', 'test', 'annotations.yaml'),
{'path': '/reality/test', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'reality', 'train', 'annotations.yaml'), 'reality/train/annotations.yaml')
download_file(os.path.join(DATA_FOLDER, 'reality', 'test', 'annotations.yaml'), 'reality/test/annotations.yaml')
elif args.real:
tmp_file = os.path.join(DATA_FOLDER, 'reality.zip')
params = {
'path': '/',
'files': 'reality',
}
folder = DATA_FOLDER
approx_size = 11
download(tmp_file, params, folder, approx_size)
download_and_extract_zip(tmp_file, 'reality.zip', folder, approx_size)
elif args.simulation and args.test and args.annotations:
download_file(os.path.join(DATA_FOLDER, 'simulation', 'test', 'annotations.yaml'),
{'path': '/simulation/test', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'simulation', 'test', 'annotations.yaml'), 'simulation/test/annotations.yaml')
elif args.simulation and args.test:
tmp_file = os.path.join(DATA_FOLDER, 'simulation', 'test.zip')
params = {
'path': '/simulation',
'files': 'test',
}
folder = os.path.join(DATA_FOLDER, 'simulation')
folder = os.path.join(DATA_FOLDER, 'simulation/test/test.zip')
approx_size = 28
download(tmp_file, params, folder, approx_size)
download_and_extract_zip(tmp_file, 'simulation/test.zip', folder, approx_size)
elif args.simulation and args.train and args.annotations:
download_file(os.path.join(DATA_FOLDER, 'simulation', 'train', 'annotations.yaml'),
{'path': '/simulation/train', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'simulation', 'train', 'annotations.yaml'), 'simulation/train/annotations.yaml')
elif args.simulation and args.train:
tmp_file = os.path.join(DATA_FOLDER, 'simulation', 'train.zip')
params = {
'path': '/simulation',
'files': 'train',
}
folder = os.path.join(DATA_FOLDER, 'simulation')
approx_size = 152
download(tmp_file, params, folder, approx_size)
download_and_extract_zip(tmp_file, 'simulation/train.zip', folder, approx_size)
elif args.simulation and args.annotations:
download_file(os.path.join(DATA_FOLDER, 'simulation', 'train', 'annotations.yaml'),
{'path': '/simulation/train', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'simulation', 'test', 'annotations.yaml'),
{'path': '/simulation/test', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'simulation', 'train', 'annotations.yaml'), 'simulation/train/annotations.yaml')
download_file(os.path.join(DATA_FOLDER, 'simulation', 'test', 'annotations.yaml'), 'simulation/test/annotations.yaml')
elif args.simulation:
tmp_file = os.path.join(DATA_FOLDER, 'simulation.zip')
params = {
'path': '/',
'files': 'simulation',
}
folder = DATA_FOLDER
approx_size = 180
download(tmp_file, params, folder, approx_size)
download_and_extract_zip(tmp_file, 'simulation.zip', folder, approx_size)
elif args.annotations:
download_file(os.path.join(DATA_FOLDER, 'reality', 'train', 'annotations.yaml'),
{'path': '/reality/train', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'reality', 'test', 'annotations.yaml'),
{'path': '/reality/test', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'simulation', 'train', 'annotations.yaml'),
{'path': '/simulation/train', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'simulation', 'test', 'annotations.yaml'),
{'path': '/simulation/test', 'files': 'annotations.yaml'})
download_file(os.path.join(DATA_FOLDER, 'reality', 'train', 'annotations.yaml'), 'reality/train/annotations.yaml')
download_file(os.path.join(DATA_FOLDER, 'reality', 'test', 'annotations.yaml'), 'reality/test/annotations.yaml')
download_file(os.path.join(DATA_FOLDER, 'simulation', 'train', 'annotations.yaml'), 'simulation/train/annotations.yaml')
download_file(os.path.join(DATA_FOLDER, 'simulation', 'test', 'annotations.yaml'), 'simulation/test/annotations.yaml')

0 comments on commit 9325d91

Please sign in to comment.