diff --git a/bano/sources/cadastre_ld.py b/bano/sources/cadastre_ld.py index 42240d2..e6ed77b 100644 --- a/bano/sources/cadastre_ld.py +++ b/bano/sources/cadastre_ld.py @@ -13,6 +13,7 @@ from ..constants import DEPARTEMENTS from .. import db from .. import helpers as hp +from .. import batch as b def process(departements, **kwargs): @@ -23,10 +24,11 @@ def process(departements, **kwargs): for dept in sorted(departements): print(f"Processing {dept}") status = download(dept) - # if status: - import_to_pg(dept) + if status: + import_to_pg(dept) def download(departement): + id_batch = b.batch_start_log("download source", "LD CADASTRE", departement) destination = get_destination(departement) headers = {} if destination.exists(): @@ -41,10 +43,18 @@ def download(departement): f.write(resp.content) mtime = parsedate_to_datetime(resp.headers["Last-Modified"]).timestamp() os.utime(destination, (mtime, mtime)) + b.batch_stop_log(id_batch, True) return True + if resp.status_code == 304: + b.batch_stop_log(id_batch, True) + return False + + print(f"Code de téléchargement : {resp.status_code}") + b.batch_stop_log(id_batch, False) return False def import_to_pg(departement, **kwargs): + id_batch = b.batch_start_log("import source", "LD CADASTRE", departement) fichier_source = get_destination(departement) with gzip.open(fichier_source, mode="rt") as f: json_source = json.load(f) @@ -62,9 +72,11 @@ def import_to_pg(departement, **kwargs): ) if a_values: cur_insert.execute(str_query + ",".join(a_values) + ";COMMIT;") + b.batch_stop_log(id_batch, True) except psycopg2.DataError as e: print(e) db.bano_db.reset() + b.batch_stop_log(id_batch, False) def get_destination(departement): try: