Add notification via email

main.py

@@ -2,15 +2,29 @@
 import settings
 import storage
 import wrangling
+import notification
+from collections import defaultdict
 
 print('Downloading codelist...')
 storage.download_codelist(settings.CODELIST_FILENAME)
 
+uploaded_files = defaultdict(list)
+
 for dataset_url in scraping.get_datasets_urls(settings.TARGET_URL):
     print(f'Downloading {dataset_url}...')
     dataset = scraping.download_dataset(dataset_url)
+
     print(f'Staging {dataset}...')
     output_files = wrangling.stage_dataset(dataset)
+
     print('Uploading output files...')
     for file in output_files:
-        storage.upload(file)
+        gdrive_file = storage.upload(file)
+        download_url = gdrive_file['webContentLink']
+        filename = file.name
+        uploaded_files[dataset.stem].append((filename, download_url))
+        print(f"{filename} -> {download_url}")
+    print()
+
+print('Notifying results...')
+notification.notify(uploaded_files)

notification.py

@@ -0,0 +1,37 @@
+import smtplib
+from email.message import EmailMessage
+
+import settings
+
+
+def notify(uploaded_files):
+    buff = []
+    for stem, files in uploaded_files.items():
+        buff.append('<p>')
+        buff.append(f'<b>{stem}</b><br>')
+        for file, url in files:
+            buff.append(f'<a href="{url}">{file}</a><br>')
+        buff.append('</p>')
+    uploaded_files = '\n'.join(buff)
+    subject = '📊 Actualización de datos SIEMAC'
+    content = f'''<p>Hola,</p>
+<p>Se ha realizado la subida de datos SIEMAC tras el scraping a EUROSTAT y su posterior
+procesamiento.</p>
+<p>Relación de ficheros con sus URLs de descarga:</p>
+
+{uploaded_files}
+
+Saludos,<br>
+El equipo de informática.
+'''
+
+    msg = EmailMessage()
+    msg.set_content(content, subtype='html')
+    msg['Subject'] = subject
+    msg['From'] = settings.NOTIFICATION_FROM_ADDR
+    msg['To'] = settings.NOTIFICATION_TO_ADDRS
+
+    s = smtplib.SMTP(settings.SMTP_SERVER, port=settings.SMTP_PORT)
+    s.login(settings.SMTP_USERNAME, settings.SMTP_PASSWORD)
+    s.send_message(msg)
+    s.quit()

settings.py

@@ -22,3 +22,10 @@
 GDRIVE_API_CREDENTIALS = config('GDRIVE_API_CREDENTIALS', default='gdrive-credentials.json')
 GDRIVE_API_SECRETS = config('GDRIVE_API_SECRETS', default='gdrive-secrets.json')
 GDRIVE_UPLOAD_FOLDER_ID = config('GDRIVE_UPLOAD_FOLDER_ID')
+
+NOTIFICATION_FROM_ADDR = config('NOTIFICATION_FROM_ADDR')
+NOTIFICATION_TO_ADDRS = config('NOTIFICATION_TO_ADDRS', cast=config.list)
+SMTP_SERVER = config('SMTP_SERVER')
+SMTP_PORT = config('SMTP_PORT')
+SMTP_USERNAME = config('SMTP_USERNAME')
+SMTP_PASSWORD = config('SMTP_PASSWORD')

storage.py

@@ -7,7 +7,7 @@
 
 
 def upload(filepath):
-    drive.put(filepath, overwrite=True)
+    return drive.put(filepath, overwrite=True)
 
 
 def download_codelist(filepath):