web_mapper.py

import Queue
import threading
import os
import urllib2

threads = 10
target = "http://www.test.com"
directory = "/Users/<USERNAME>/Downloads/joomla3-1.1"
filters = [".jpg", ".gif", "png", ".css"]

os.chdir(directory)
web_paths = Queue.Queue()

for r,d,f in os.walk("."):
    for files in f:
        remote_path = f"{r}/{files}"
        if remote_path.startswith("."):
            remote_path = remote_path[1:]
        if os.path.splitext(files)[1] not in filters:
            web_paths.put(remote_path)

def test_remote():
    while not web_paths.empty(): 
        path = web_paths.get()
        request = urllib2.Request(f"{target}{path}")
        try:
            response = urllib2.urlopen(request)
            content  = response.read()
            print(f"[{response.code}] => {path}") 
            response.close()
        except urllib2.HTTPError as error: 
            print(f"Failed {error.code}")
            pass
        
for i in range(threads): 
    print(f"Spawning thread: {i}")
    t = threading.Thread(target=test_remote)
    t.start()