Implement importing of extracts to database

Dockerfile

@@ -1,16 +1,33 @@
 FROM docker.io/debian:buster-slim
 
 # install dependencies
+ARG PG_VERSION=11
+ENV PG_VERSION=${PG_VERSION}
 RUN apt update
-RUN apt install -y --no-install-recommends pipenv osmctools rsync
+RUN apt install -y --no-install-recommends python3 python pipenv python3-setuptools python3-dev python-wheel \
+    python-setuptools python-pip rsync postgresql-${PG_VERSION} osmctools osm2pgsql git libpq-dev gcc make unzip \
+    postgis curl
+RUN apt install -y postgresql-${PG_VERSION}-postgis
+
+RUN pg_dropcluster --stop ${PG_VERSION} main
+
+# add dependency sources
+RUN git clone -b v1.8.0 https://github.com/mapzen/vector-datasource.git /app/src/vector-datasource
+RUN pip install -r /app/src/vector-datasource/requirements.txt
+ENV PYTHONPATH=${PYTHONPATH}:/app/src/vector-datasource
+RUN apt purge -y python-setuptools python-pip
 
 # add sources
-ADD Pipfile Pipfile.lock /app/src/
-WORKDIR /app/src
-RUN pipenv install --system --deploy --ignore-pipfile
-ADD generate_extracts.py /app/src/
+ADD Pipfile Pipfile.lock /app/src/osm_tile_data_extract/
+WORKDIR /app/src/osm_tile_data_extract
+RUN pipenv install --system --deploy
+RUN apt purge -y libpq-dev gcc python3-dev
+RUN apt autoremove -y
+
+ADD main.py /app/src/osm_tile_data_extract
+ADD osm_tile_data_extract /app/src/osm_tile_data_extract/osm_tile_data_extract
 
 # add image metadata
 VOLUME /app/tmp
 VOLUME /app/out
-ENTRYPOINT ["/app/src/generate_extracts.py", "-w", "/app/tmp", "-o", "/app/out"]
+ENTRYPOINT ["/app/src/osm_tile_data_extract/main.py", "-w", "/app/tmp", "-o", "/app/out"]

Pipfile

@@ -1,9 +1,15 @@
+[requires]
+python_version = "3"
+
 [[source]]
 name = "pypi"
 url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+ipython = "*"
 
 [packages]
 mercantile = "*"
+swaggerpy = "*"
+pyswagger = "*"

main.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+
+from pathlib import Path
+from osm_tile_data_extract.generate_extracts import Program as GenerateExtracts
+from osm_tile_data_extract.import_extracts import Program as ImportExtracts
+
+
+def parse_args() -> argparse.Namespace:
+    def directory_type(raw: str):
+        p = Path(raw)
+        p.mkdir(exist_ok=True)
+        if not p.is_dir():
+            raise argparse.ArgumentTypeError(f'Path {raw} is not a directory')
+        return p
+
+    def auth_type(raw: str) -> list:
+        if ':' not in raw or len(raw.split(':')) != 2:
+            raise argparse.ArgumentTypeError('Authentication has invalid format')
+        return raw.split(':')
+
+    parser = argparse.ArgumentParser('osm-tile-data-extract')
+    parser.add_argument('-w', '--working-dir', dest='working_dir', type=directory_type,
+                        default=os.path.join(os.path.dirname(__file__), 'tmp'),
+                        help='Working directory in which intermediate and temporary files are stored')
+    parser.add_argument('-o', '--output-dir', dest='output_dir', type=directory_type,
+                        default=os.path.join(os.path.dirname(__file__), 'out'))
+    parser.add_argument('--mapping-url', dest='mapping_url', type=str, required=True,
+                        help='Base URL under which a tileserver-mapping server is reachable')
+    parser.add_argument('--mapping-auth', dest='mapping_auth', type=auth_type, required=True,
+                        help='<username>:<password> combination used to authenticate at the tileserver-mapping')
+
+    sub_parsers = parser.add_subparsers(dest='command')
+    GenerateExtracts.add_args(sub_parsers.add_parser('generate-extracts',
+                                                     description='Extract similarly sized files from  the latest'
+                                                                 'OpenStreetMap planet dump'))
+    ImportExtracts.add_args(sub_parsers.add_parser('import-extracts',
+                                                   description='Import one pbf extract into postgresql and afterwards'
+                                                               'dump the database'))
+
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    if args.command == 'generate-extracts':
+        program = GenerateExtracts(args)
+    elif args.command == 'import-extracts':
+        program = ImportExtracts(args)
+
+    program.run()