main.py

import argparse
from concurrent.futures import ThreadPoolExecutor
from crawler_env import WebEnv
from agent import CrawlerAgent
from utils import log_session, reflect_and_improve

parser = argparse.ArgumentParser()
parser.add_argument('--goal', type=str, required=True, help='Crawling goal')
parser.add_argument('--start_url', type=str, required=True, help='Starting URL')
parser.add_argument('--parallel', type=int, default=1, help='Parallel crawls')
args = parser.parse_args()

# LLM-native planning
agent = CrawlerAgent(args.goal)
sub_urls = agent.plan_sub_urls(args.start_url, args.parallel)

# Parallel execution
with ThreadPoolExecutor(max_workers=args.parallel) as executor:
    envs = [WebEnv(url, args.goal) for url in sub_urls]
    futures = [executor.submit(agent.crawl, e) for e in envs]
    results = [f.result() for f in futures]

log_session(results)
improvements = reflect_and_improve(results)
print("Suggested improvements:", improvements)