Gestion des batchs de telechargement de la source LD Cadastre

bano/sources/cadastre_ld.py

@@ -13,6 +13,7 @@
 from ..constants import DEPARTEMENTS
 from .. import db
 from .. import helpers as hp
+from .. import batch as b
 
 
 def process(departements, **kwargs):
@@ -23,10 +24,11 @@ def process(departements, **kwargs):
     for dept in sorted(departements):
         print(f"Processing {dept}")
         status = download(dept)
-        # if status:
-        import_to_pg(dept)
+        if status:
+            import_to_pg(dept)
 
 def download(departement):
+    id_batch = b.batch_start_log("download source", "LD CADASTRE", departement)
     destination = get_destination(departement)
     headers = {}
     if destination.exists():
@@ -41,10 +43,18 @@ def download(departement):
             f.write(resp.content)
         mtime = parsedate_to_datetime(resp.headers["Last-Modified"]).timestamp()
         os.utime(destination, (mtime, mtime))
+        b.batch_stop_log(id_batch, True)
         return True
+    if resp.status_code == 304:
+        b.batch_stop_log(id_batch, True)
+        return False
+
+    print(f"Code de téléchargement : {resp.status_code}")
+    b.batch_stop_log(id_batch, False)
     return False
 
 def import_to_pg(departement, **kwargs):
+    id_batch = b.batch_start_log("import source", "LD CADASTRE", departement)
     fichier_source = get_destination(departement)
     with gzip.open(fichier_source, mode="rt") as f:
         json_source = json.load(f)
@@ -62,9 +72,11 @@ def import_to_pg(departement, **kwargs):
                     )
                 if a_values:
                     cur_insert.execute(str_query + ",".join(a_values) + ";COMMIT;")
+                b.batch_stop_log(id_batch, True)
             except psycopg2.DataError as e:
                 print(e)
                 db.bano_db.reset()
+                b.batch_stop_log(id_batch, False)
 
 def get_destination(departement):
     try: