Parameter to specify encoding error response

core/mirror.py

@@ -1,7 +1,7 @@
 import os
 
 
-def mirror(url, response):
+def mirror(url, response,encoding_error_response):
     if response != 'dummy':
         clean_url = url.replace('http://', '').replace('https://', '').rstrip('/')
         parts = clean_url.split('?')[0].split('/')
@@ -36,4 +36,4 @@ def mirror(url, response):
         if len(url.split('?')) > 1:
             trail += '?' + url.split('?')[1]
         with open(path + name + trail, 'w+') as out_file:
-            out_file.write(response.encode('utf-8'))
+            out_file.write(response.encode('utf-8',errors=encoding_error_response))

core/utils.py

@@ -75,14 +75,14 @@ def remove_regex(urls, regex):
     return non_matching_urls
 
 
-def writer(datasets, dataset_names, output_dir):
+def writer(datasets, dataset_names, output_dir,encoding_error_response):
     """Write the results."""
     for dataset, dataset_name in zip(datasets, dataset_names):
         if dataset:
             filepath = output_dir + '/' + dataset_name + '.txt'
             with open(filepath, 'w+') as out_file:
                 joined = '\n'.join(dataset)
-                out_file.write(str(joined.encode('utf-8').decode('utf-8')))
+                out_file.write(str(joined.encode('utf-8',errors=encoding_error_response).decode('utf-8')))
                 out_file.write('\n')
 
 
@@ -98,12 +98,12 @@ def timer(diff, processed):
     return minutes, seconds, time_per_request
 
 
-def entropy(string):
+def entropy(string,encoding_error_response):
     """Calculate the entropy of a string."""
     entropy = 0
     for number in range(256):
-        result = float(string.encode('utf-8').count(
-            chr(number))) / len(string.encode('utf-8'))
+        result = float(string.encode('utf-8',errors=encoding_error_response).count(
+            chr(number))) / len(string.encode('utf-8',errors=encoding_error_response))
         if result != 0:
             entropy = entropy - result * math.log(result, 2)
     return entropy

photon.py

@@ -80,7 +80,7 @@
                     type=float)
 parser.add_argument('-p', '--proxy', help='Proxy server IP:PORT or DOMAIN:PORT', dest='proxies',
                     type=proxy_type)
-
+parser.add_argument('--encoding-error', help='encoding error response parameter', dest='encoding_error',default='strict',choices=['backslashreplace','ignore','namereplace','strict','replace','xmlcharrefreplace',])
 # Switches
 parser.add_argument('--clone', help='clone the website locally', dest='clone',
                     action='store_true')
@@ -190,6 +190,7 @@
 host = urlparse(main_url).netloc
 
 output_dir = args.output or host
+encoding_error_response = args.encoding_error
 
 try:
     domain = top_level(main_url)
@@ -240,7 +241,7 @@ def extractor(url):
     """Extract details from the response body."""
     response = requester(url, main_url, delay, cook, headers, timeout, host, proxies, user_agents, failed, processed)
     if clone:
-        mirror(url, response)
+        mirror(url, response,encoding_error_response)
     matches = rhref.findall(response)
     for link in matches:
         # Remove everything after a "#" to deal with in-page anchors
@@ -282,7 +283,7 @@ def extractor(url):
     if api:
         matches = rentropy.findall(response)
         for match in matches:
-            if entropy(match) >= 4:
+            if entropy(match,encoding_error_response) >= 4:
                 verb('Key', match)
                 keys.add(url + ': ' + match)
 
@@ -382,7 +383,7 @@ def jscanner(url):
 dataset_names = ['files', 'intel', 'robots', 'custom', 'failed', 'internal',
                  'scripts', 'external', 'fuzzable', 'endpoints', 'keys']
 
-writer(datasets, dataset_names, output_dir)
+writer(datasets, dataset_names, output_dir,encoding_error_response)
 # Printing out results
 print(('%s-%s' % (red, end)) * 50)
 for dataset, dataset_name in zip(datasets, dataset_names):
@@ -407,7 +408,7 @@ def jscanner(url):
     from plugins.find_subdomains import find_subdomains
     subdomains = find_subdomains(domain)
     print('%s %i subdomains found' % (info, len(subdomains)))
-    writer([subdomains], ['subdomains'], output_dir)
+    writer([subdomains], ['subdomains'], output_dir,encoding_error_response)
     datasets['subdomains'] = subdomains
     from plugins.dnsdumpster import dnsdumpster
     print('%s Generating DNS map' % run)