From ee602b1e7b845f28749fc0dbf48707e08caa4a56 Mon Sep 17 00:00:00 2001 From: thewhiteh4t Date: Fri, 17 May 2024 01:54:18 +0530 Subject: [PATCH] fixed domain extraction, temp disabled wayback in subdomain enum, faster crt.sh lookup, fixed typo --- finalrecon.py | 54 +++++++++++++++------- modules/crawler.py | 25 ++-------- modules/subdom.py | 4 +- modules/subdomain_modules/certspot_subs.py | 2 +- modules/subdomain_modules/crtsh_subs.py | 39 ++++++++-------- modules/whois.py | 2 +- 6 files changed, 64 insertions(+), 62 deletions(-) diff --git a/finalrecon.py b/finalrecon.py index cbfbc68..d76e3a9 100644 --- a/finalrecon.py +++ b/finalrecon.py @@ -106,6 +106,7 @@ import ipaddress import tldextract from json import loads, dumps +from urllib import parse type_ip = False data = {} @@ -173,25 +174,44 @@ def save_key(key_string): target = target[:-1] print(f'{G}[+] {C}Target : {W}{target}') - ext = tldextract.extract(target) - domain = ext.registered_domain - if not domain: - domain = ext.domain - domain_suffix = ext.suffix - - if ext.subdomain: - hostname = f'{ext.subdomain}.{ext.domain}.{ext.suffix}' + + split_url = parse.urlsplit(target) + extractor = tldextract.TLDExtract() + parsed_url = extractor.extract_urllib(split_url) + protocol = split_url.scheme + + if split_url.port: + if not parsed_url.subdomain: + netloc = parsed_url.domain # localhost:8000 + domain = netloc.split(':')[0] + domain_suffix = '' + hostname = domain + else: + netloc = f'{parsed_url.subdomain}.{parsed_url.domain}' # abc.com:8000 + domain = parsed_url.subdomain + domain_suffix = parsed_url.domain.split(':')[0] + hostname = f'{domain}.{domain_suffix}' else: - hostname = domain + if len(parsed_url.registered_domain) == 0: + netloc = parsed_url.domain # 8.8.8.8 + domain = '' + domain_suffix = '' + else: + netloc = parsed_url.registered_domain # abc.com + domain = parsed_url.domain + domain_suffix = parsed_url.suffix + hostname = netloc try: ipaddress.ip_address(hostname) type_ip = True ip = hostname + private_ip = ipaddress.ip_address(ip).is_private except Exception: try: ip = socket.gethostbyname(hostname) print(f'\n{G}[+] {C}IP Address : {W}{str(ip)}') + private_ip = ipaddress.ip_address(ip).is_private except Exception as e: print(f'\n{R}[-] {C}Unable to Get IP : {W}{str(e)}') sys.exit(1) @@ -228,11 +248,11 @@ def save_key(key_string): headers(target, out_settings, data) cert(hostname, sslp, out_settings, data) whois_lookup(domain, domain_suffix, path_to_script, out_settings, data) - dnsrec(domain, out_settings, data) - if not type_ip: - subdomains(domain, tout, out_settings, data, conf_path) + dnsrec(hostname, out_settings, data) + if not type_ip and not private_ip: + subdomains(hostname, tout, out_settings, data, conf_path) scan(ip, out_settings, data, pscan_threads) - crawler(target, out_settings, data) + crawler(target, protocol, netloc, out_settings, data) hammer(target, threads, tout, wdlist, redir, sslv, dserv, out_settings, data, filext) timetravel(target, data, out_settings) @@ -254,17 +274,17 @@ def save_key(key_string): if crawl: from modules.crawler import crawler log_writer('Starting crawler...') - crawler(target, out_settings, data) + crawler(target, protocol, netloc, out_settings, data) if dns: from modules.dns import dnsrec log_writer('Starting DNS enum...') - dnsrec(domain, out_settings, data) + dnsrec(hostname, out_settings, data) - if subd and not type_ip: + if subd and not type_ip and not private_ip: from modules.subdom import subdomains log_writer('Starting subdomain enum...') - subdomains(domain, tout, out_settings, data, conf_path) + subdomains(hostname, tout, out_settings, data, conf_path) elif subd and type_ip: print(f'{R}[-] {C}Sub-Domain Enumeration is Not Supported for IP Addresses{W}\n') diff --git a/modules/crawler.py b/modules/crawler.py index 6d63cfc..cb0d87b 100644 --- a/modules/crawler.py +++ b/modules/crawler.py @@ -31,14 +31,14 @@ sm_crawl_total = [] -def crawler(target, output, data): +def crawler(target, protocol, netloc, output, data): global r_url, sm_url print(f'\n{Y}[!] Starting Crawler...{W}\n') try: rqst = requests.get(target, headers=user_agent, verify=False, timeout=10) except Exception as exc: - print(f'{R} [-] Exception : {C}{exc}{W}') + print(f'{R}[-] Exception : {C}{exc}{W}') log_writer(f'[crawler] Exception = {exc}') return @@ -47,24 +47,9 @@ def crawler(target, output, data): page = rqst.content soup = bs4.BeautifulSoup(page, 'lxml') - protocol = target.split('://') - protocol = protocol[0] - temp_tgt = target.split('://')[1] - pattern = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}' - custom = bool(re.match(pattern, temp_tgt)) - if custom: - r_url = f'{protocol}://{temp_tgt}/robots.txt' - sm_url = f'{protocol}://{temp_tgt}/sitemap.xml' - base_url = f'{protocol}://{temp_tgt}' - else: - ext = tldextract.extract(target) - if ext.subdomain: - hostname = f'{ext.subdomain}.{ext.domain}.{ext.suffix}' - else: - hostname = ext.registered_domain - base_url = f'{protocol}://{hostname}' - r_url = f'{base_url}/robots.txt' - sm_url = f'{base_url}/sitemap.xml' + r_url = f'{protocol}://{netloc}/robots.txt' + sm_url = f'{protocol}://{netloc}/sitemap.xml' + base_url = f'{protocol}://{netloc}' loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) diff --git a/modules/subdom.py b/modules/subdom.py index 3df2839..a2018f8 100644 --- a/modules/subdom.py +++ b/modules/subdom.py @@ -36,9 +36,9 @@ async def query(hostname, tout, conf_path): virust(hostname, conf_path, session), shodan(hostname, conf_path, session), certspot(hostname, session), - machine(hostname, session), + #machine(hostname, session), hackertgt(hostname, session), - crtsh(hostname) + crtsh(hostname, session) ) await session.close() diff --git a/modules/subdomain_modules/certspot_subs.py b/modules/subdomain_modules/certspot_subs.py index 076b6f7..a05b61a 100644 --- a/modules/subdomain_modules/certspot_subs.py +++ b/modules/subdomain_modules/certspot_subs.py @@ -26,7 +26,7 @@ async def certspot(hostname, session): if status == 200: json_data = await resp.text() json_read = loads(json_data) - print(f'{G}[+] {Y}Certsport {W}found {C}{len(json_read)} {W}subdomains!') + print(f'{G}[+] {Y}Certspotter {W}found {C}{len(json_read)} {W}subdomains!') for i in range(0, len(json_read)): domains = json_read[i]['dns_names'] parent.found.extend(domains) diff --git a/modules/subdomain_modules/crtsh_subs.py b/modules/subdomain_modules/crtsh_subs.py index 5c92e49..092c78a 100644 --- a/modules/subdomain_modules/crtsh_subs.py +++ b/modules/subdomain_modules/crtsh_subs.py @@ -6,32 +6,29 @@ W = '\033[0m' # white Y = '\033[33m' # yellow -import psycopg2 +from json import loads import modules.subdom as parent from modules.write_log import log_writer - -async def crtsh(hostname): +async def crtsh(hostname, session): print(f'{Y}[!] {C}Requesting {G}crt.sh{W}') + url = f'https://crt.sh/?dNSName=%25.{hostname}&output=json' + try: - conn = psycopg2.connect( - host="crt.sh", - database="certwatch", - user="guest", - port="5432" - ) - conn.autocommit = True - cur = conn.cursor() - query = f"SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'dNSName' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%.{hostname}'))" - cur.execute(query) - result = cur.fetchall() - cur.close() - conn.close() - tmp_list = [] - for url in result: - tmp_list.append(url[0]) - print(f'{G}[+] {Y}CRT.sh {W}found {C}{len(tmp_list)} {W}subdomains!') - parent.found.extend(tmp_list) + async with session.get(url) as resp: + status = resp.status + if status == 200: + data = await resp.text() + data_json = loads(data) + tmp_list = [] + for entry in data_json: + subdomain = entry['name_value'] + tmp_list.append(subdomain) + print(f'{G}[+] {Y}crt.sh {W}found {C}{len(tmp_list)} {W}subdomains!') + parent.found.extend(tmp_list) + else: + print(f'{R}[-] {C}HackerTarget Status : {W}{status}') + log_writer(f'[htarget_subs] Status = {status}, expected 200') except Exception as exc: print(f'{R}[-] {C}crtsh Exception : {W}{exc}') log_writer(f'[crtsh_subs] Exception = {exc}') diff --git a/modules/whois.py b/modules/whois.py index ddca88a..1f54379 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -45,7 +45,7 @@ def whois_lookup(domain, tld, script_path, output, data): try: whois_sv = db_json[tld] - whois_info = asyncio.run(get_whois(domain, whois_sv)) + whois_info = asyncio.run(get_whois(f'{domain}.{tld}', whois_sv)) print(whois_info['whois']) result.update(whois_info) except KeyError: