Merge pull request #341 from coderxio/jrlegrand/fix-fda-headers

Add headers to requests
coderxio · Jan 15, 2025 · 5c1a981 · 5c1a981
2 parents 6b857e7 + 7b3273d
commit 5c1a981
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/airflow/dags/sagerx.py b/airflow/dags/sagerx.py
@@ -41,7 +41,11 @@ def download_dataset(url: str, dest: Path = Path.cwd(), file_name: str = None):
     import requests
     import re
 
-    with requests.get(url, stream=True, allow_redirects=True) as r:
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    }
+
+    with requests.get(url, stream=True, allow_redirects=True, headers=headers) as r:
         r.raise_for_status()
 
         if file_name == None: