adjusted for new tldextract version

thewhiteh4t · Oct 19, 2023 · 93f4182 · 93f4182
1 parent 10d82c3
commit 93f4182
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/modules/crawler.py b/modules/crawler.py
@@ -58,7 +58,10 @@ def crawler(target, output, data):
 			base_url = f'{protocol}://{temp_tgt}'
 		else:
 			ext = tldextract.extract(target)
-			hostname = '.'.join(part for part in ext if part)
+			if ext.subdomain:
+				hostname = f'{ext.subdomain}.{ext.domain}.{ext.suffix}'
+			else:
+				hostname = ext.registered_domain
 			base_url = f'{protocol}://{hostname}'
 			r_url = f'{base_url}/robots.txt'
 			sm_url = f'{base_url}/sitemap.xml'