Incorporated screenshot search into background link parsing

Author: CodexVeritas

.devcontainer/postinstall.sh

@@ -22,3 +22,7 @@ source ../.venv/bin/activate
 
 # Show which Python interpreter is being used
 which python
+
+# Install playwright
+playwright install
+playwright install-deps

.vscode/settings.json

@@ -3,7 +3,7 @@
   "python.testing.unittestArgs": [
     "-v",
     "-s",
-    "./tests",
+    "./code_tests",
     "-p",
     "test_*.py"
   ],

README.md

@@ -9,6 +9,12 @@ Last Update: Nov 3 2024
 # Quick Install
 Install this package with `pip install forecasting-tools`
 
+If you want to use advanced SmartSearcher functionality, you might also need to run:
+```bash
+playwright install
+playwright install-deps
+```
+
 # Overview
 Demo website: https://mokoresearch.streamlit.app/
 

code_tests/low_cost_or_live_api_tests/test_forecast_database_manager.py

@@ -58,15 +58,15 @@ async def test_base_rate_report_can_be_added_to_coda() -> None:
 
 
 def get_forecast_example_reports() -> list[BinaryReport]:
-    metaculus_data_path = "tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/metaculus_forecast_report_examples.json"
+    metaculus_data_path = "code_tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/metaculus_forecast_report_examples.json"
     metaculus_reports = BinaryReport.convert_project_file_path_to_object_list(
         metaculus_data_path
     )
     return metaculus_reports
 
 
 def get_base_rate_example_reports() -> list[BaseRateReport]:
-    base_rate_data_path = "tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/base_rate_reports.json"
+    base_rate_data_path = "code_tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/base_rate_reports.json"
     base_rate_reports = (
         BaseRateReport.convert_project_file_path_to_object_list(
             base_rate_data_path

code_tests/low_cost_or_live_api_tests/test_smart_searcher.py

@@ -64,9 +64,19 @@ async def test_ask_question_empty_prompt() -> None:
 
 async def test_screenshot_question() -> None:
     with MonetaryCostManager() as cost_manager:
-        searcher = SmartSearcher(num_sites_to_screenshot=1)
-        question = "When was the most noticeable recent dip in the graph from https://fred.stlouisfed.org/series/GDPC1? Say 0 if you do not know."
+        searcher = SmartSearcher(num_sites_to_deep_dive=2)
+        question = "When was the most noticeable recent dip in the graph from https://fred.stlouisfed.org/series/GDPC1? Say 0 if you do not know. Please search specifically for the site itself."
         result = await searcher.invoke(question)
         logger.info(f"Result: {result}")
         logger.info(f"Cost: {cost_manager.current_usage}")
         assert "2020" in result
+
+
+@pytest.mark.skip("Run this when needed as it's purely a qualitative test")
+async def test_screenshot_question_2() -> None:
+    with MonetaryCostManager() as cost_manager:
+        searcher = SmartSearcher(num_sites_to_deep_dive=2)
+        question = "Please tell me about the recent trends in the Federal Funds Effective Rate."
+        result = await searcher.invoke(question)
+        logger.info(f"Result: {result}")
+        logger.info(f"Cost: {cost_manager.current_usage}")

code_tests/low_cost_or_live_api_tests/test_url_scraper.py

@@ -7,7 +7,9 @@ async def test_get_screenshot_as_file() -> None:
     url_scraper = UrlScraper()
     test_url = "https://example.com"
 
-    screenshot_file = await url_scraper.get_screenshot_of_url_as_file(test_url)
+    screenshot_file = await url_scraper.get_screenshot_of_url_as_image(
+        test_url
+    )
     assert isinstance(screenshot_file, Image.Image)
     assert screenshot_file.width > 0
     assert screenshot_file.height > 0

code_tests/no_cost_expect_all_to_succeed/test_forecasting/test_forecast_report.py

@@ -16,7 +16,7 @@
 
 def test_metaculus_report_is_jsonable() -> None:
     temp_writing_path = "temp/temp_metaculus_report.json"
-    read_report_path = "tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/metaculus_forecast_report_examples.json"
+    read_report_path = "code_tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/metaculus_forecast_report_examples.json"
     jsonable_assertations.assert_reading_and_printing_from_file_works(
         BinaryReport, read_report_path, temp_writing_path
     )

code_tests/no_cost_expect_all_to_succeed/test_forecasting/test_metaculus_question.py

@@ -6,7 +6,7 @@
 
 def test_metaculus_question_is_jsonable() -> None:
     temp_writing_path = "temp/temp_metaculus_question.json"
-    read_report_path = "tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/metaculus_questions.json"
+    read_report_path = "code_tests/no_cost_expect_all_to_succeed/test_forecasting/forecasting_test_data/metaculus_questions.json"
     jsonable_assertations.assert_reading_and_printing_from_file_works(
         BinaryQuestion, read_report_path, temp_writing_path
     )

forecasting_tools/forecasting/forecast_team/final_decision_agent.py

@@ -154,9 +154,7 @@ async def __get_research_summary_and_populate_if_empty(self) -> str:
         summary_markdown = await model.invoke(prompt)
         cleaned_summary_markdown = strip_code_block_markdown(summary_markdown)
         self.__research_summary = cleaned_summary_markdown
-        logger.info(
-            f"Made research summary: {cleaned_summary_markdown[:100]}..."
-        )
+        logger.info("Made research summary for final decision agent")
         return cleaned_summary_markdown
 
     async def __create_unified_explanation(

forecasting_tools/forecasting/forecast_team/research_coordinator.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
 import logging
+import re
 
 from forecasting_tools.ai_models.ai_utils.ai_misc import clean_indents
 from forecasting_tools.forecasting.helpers.configured_llms import BasicLlm
+from forecasting_tools.forecasting.helpers.url_scraper import UrlScraper
 from forecasting_tools.forecasting.questions_and_reports.metaculus_question import (
     MetaculusQuestion,
 )
@@ -21,6 +23,9 @@
     QuestionRouter,
 )
 from forecasting_tools.util import async_batching
+from forecasting_tools.util.async_batching import (
+    run_coroutines_while_removing_and_logging_exceptions,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -39,17 +44,71 @@ async def create_full_markdown_research_report(
         num_base_rate_questions: int,
         num_base_rate_questions_with_deep_research: int,
     ) -> str:
+        question_details_markdown = (
+            await self.__generate_question_details_markdown()
+        )
         background_markdown = await self.generate_background_markdown(
-            num_of_background_questions
+            num_of_background_questions, question_details_markdown
         )
         base_rate_markdown = await self.generate_base_rate_markdown(
             num_base_rate_questions,
             num_base_rate_questions_with_deep_research,
             background_markdown,
         )
-        combined_markdown = background_markdown + "\n\n" + base_rate_markdown
+        combined_markdown = (
+            question_details_markdown
+            + "\n\n"
+            + background_markdown
+            + "\n\n"
+            + base_rate_markdown
+        )
         return combined_markdown
 
+    async def __generate_question_details_markdown(self) -> str:
+        question_details = self.question.give_question_details_as_markdown()
+        urls = self.__extract_urls_from_markdown_text(question_details)
+
+        if not urls:
+            return "# Question Details\nNo links found in question details."
+
+        screenshot_tasks = [
+            self.__get_screenshot_and_summary(url) for url in urls
+        ]
+
+        summaries, successful_urls = (
+            run_coroutines_while_removing_and_logging_exceptions(
+                screenshot_tasks, urls
+            )
+        )
+
+        markdown = await self.__create_question_answer_markdown_section(
+            [f"What does {url} say?" for url in successful_urls],
+            summaries,
+            question_prepend="L",
+        )
+        return markdown
+
+    def __extract_urls_from_markdown_text(self, text: str) -> list[str]:
+        markdown_link_pattern = r"\[([^\]]+)\]\(([^)]+)\)"
+        matches = re.findall(markdown_link_pattern, text)
+        return list(set(url for _, url in matches))
+
+    async def __get_screenshot_and_summary(
+        self,
+        url: str,
+    ) -> str:
+        logger.info(f"Attempting to get screenshot and summary for {url}")
+        url_scraper = UrlScraper()
+        image_data = await url_scraper.get_screenshot_of_url_as_base64(url)
+        prompt = f"You are a superforecast. Please research and write a report on the following question: {self.question.give_question_details_as_markdown()}"
+        summary = await UrlScraper.get_summary_of_screenshot(
+            image_data, prompt
+        )
+        logger.info(
+            f"Successfully got summary for {url}. Summary: {summary[:1000]}..."
+        )
+        return summary
+
     async def make_list_of_base_rate_reports(
         self,
         number_of_base_rate_reports: int,
@@ -77,10 +136,12 @@ async def make_list_of_base_rate_reports(
         return base_rate_reports
 
     async def generate_background_markdown(
-        self, num_background_questions: int
+        self,
+        num_background_questions: int,
+        additional_context: str | None = None,
     ) -> str:
         questions = await self.brainstorm_background_questions(
-            num_background_questions
+            num_background_questions, additional_context
         )
         answers = await self.answer_question_list(questions, GeneralResearcher)
         logger.info("Generated background markdown.")
@@ -134,7 +195,9 @@ async def pick_best_base_rate_questions(
         return deep_questions, shallow_questions
 
     async def brainstorm_background_questions(
-        self, num_background_questions: int
+        self,
+        num_background_questions: int,
+        additional_context: str | None = None,
     ) -> list[str]:
         logger.info(
             f"Running forecasts on question `{self.question.question_text}`"
@@ -179,6 +242,9 @@ async def brainstorm_background_questions(
             # Question Info
             {self.question.give_question_details_as_markdown()}
 
+            # Previous Research
+            {additional_context or "No previous research was provided"}
+
             # Your Turn
             Please come up with {num_background_questions} questions.
             """