Merge pull request #1472 from hurxxxx/master

feat: Add enhanced Serper API support with country, language, date filters and configurable site exclusion

Author: assafelovic

docs/docs/gpt-researcher/search-engines/search-engines.md

@@ -28,7 +28,7 @@ Thanks to our community, we have integrated the following web search engines:
 - [Google](https://developers.google.com/custom-search/v1/overview) - Env: `RETRIEVER=google`
 - [SearchApi](https://www.searchapi.io/) - Env: `RETRIEVER=searchapi`
 - [Serp API](https://serpapi.com/) - Env: `RETRIEVER=serpapi`
-- [Serper](https://serper.dev/) - Env: `RETRIEVER=serper`
+- [Serper](https://serper.dev/) - Env: `RETRIEVER=serper` - [Setup Guide](#serper)
 - [Searx](https://searx.github.io/searx/) - Env: `RETRIEVER=searx`
 - [Duckduckgo](https://pypi.org/project/duckduckgo-search/) - Env: `RETRIEVER=duckduckgo`
 - [Arxiv](https://info.arxiv.org/help/api/index.html) - Env: `RETRIEVER=arxiv`
@@ -72,4 +72,27 @@ For the custom retriever to work correctly, the response from the endpoint shoul
 
 The system assumes this response format and processes the list of sources accordingly.
 
+## Search Engine Configuration
+
+### Serper
+
+To use [Serper](https://serper.dev/) as your search engine:
+
+1. Get your API key from [serper.dev](https://serper.dev/)
+2. Set the required environment variables:
+
+```bash
+RETRIEVER=serper
+SERPER_API_KEY=your_api_key_here
+```
+
+**Optional Configuration:**
+
+```bash
+SERPER_REGION=us                    # Country code (us, kr, jp, etc.)
+SERPER_LANGUAGE=en                  # Language code (en, ko, ja, etc.)
+SERPER_TIME_RANGE=qdr:w            # Time filter (qdr:h, qdr:d, qdr:w, qdr:m, qdr:y)
+SERPER_EXCLUDE_SITES=youtube.com   # Exclude sites (comma-separated)
+```
+
 Missing a retriever? Feel free to contribute to this project by submitting issues or pull requests on our [GitHub](https://github.com/assafelovic/gpt-researcher) page.

gpt_researcher/retrievers/serper/serper.py

@@ -8,19 +8,39 @@
 
 class SerperSearch():
     """
-    Google Serper Retriever
+    Google Serper Retriever with support for country, language, and date filtering
     """
-    def __init__(self, query, query_domains=None):
+    def __init__(self, query, query_domains=None, country=None, language=None, time_range=None, exclude_sites=None):
         """
         Initializes the SerperSearch object
         Args:
             query (str): The search query string.
             query_domains (list, optional): List of domains to include in the search. Defaults to None.
+            country (str, optional): Country code for search results (e.g., 'us', 'kr', 'jp'). Defaults to None.
+            language (str, optional): Language code for search results (e.g., 'en', 'ko', 'ja'). Defaults to None.
+            time_range (str, optional): Time range filter (e.g., 'qdr:h', 'qdr:d', 'qdr:w', 'qdr:m', 'qdr:y'). Defaults to None.
+            exclude_sites (list, optional): List of sites to exclude from search results. Defaults to None.
         """
         self.query = query
         self.query_domains = query_domains or None
+        self.country = country or os.getenv("SERPER_REGION")
+        self.language = language or os.getenv("SERPER_LANGUAGE")
+        self.time_range = time_range or os.getenv("SERPER_TIME_RANGE")
+        self.exclude_sites = exclude_sites or self._get_exclude_sites_from_env()
         self.api_key = self.get_api_key()
 
+    def _get_exclude_sites_from_env(self):
+        """
+        Gets the list of sites to exclude from environment variables
+        Returns:
+            list: List of sites to exclude
+        """
+        exclude_sites_env = os.getenv("SERPER_EXCLUDE_SITES", "")
+        if exclude_sites_env:
+            # Split by comma and strip whitespace
+            return [site.strip() for site in exclude_sites_env.split(",") if site.strip()]
+        return []
+
     def get_api_key(self):
         """
         Gets the Serper API key
@@ -36,12 +56,12 @@ def get_api_key(self):
 
     def search(self, max_results=7):
         """
-        Searches the query
+        Searches the query with optional country, language, and time filtering
         Returns:
-
+            list: List of search results with title, href, and body
         """
         print("Searching with query {0}...".format(self.query))
-        """Useful for general internet search queries using the Serp API."""
+        """Useful for general internet search queries using the Serper API."""
 
         # Search the query (see https://serper.dev/playground for the format)
         url = "https://google.serper.dev/search"
@@ -51,8 +71,36 @@ def search(self, max_results=7):
             'Content-Type': 'application/json'
         }
 
-        # TODO: Add support for query domains
-        data = json.dumps({"q": self.query, "num": max_results})
+        # Build search parameters
+        query_with_filters = self.query
+
+        # Exclude sites using Google search syntax
+        if self.exclude_sites:
+            for site in self.exclude_sites:
+                query_with_filters += f" -site:{site}"
+
+        # Add domain filtering if specified
+        if self.query_domains:
+            # Add site:domain1 OR site:domain2 OR ... to the search query
+            domain_query = " site:" + " OR site:".join(self.query_domains)
+            query_with_filters += domain_query
+
+        search_params = {
+            "q": query_with_filters,
+            "num": max_results
+        }
+
+        # Add optional parameters if they exist
+        if self.country:
+            search_params["gl"] = self.country  # Geographic location (country)
+
+        if self.language:
+            search_params["hl"] = self.language  # Host language
+
+        if self.time_range:
+            search_params["tbs"] = self.time_range  # Time-based search
+
+        data = json.dumps(search_params)
 
         resp = requests.request("POST", url, timeout=10, headers=headers, data=data)
 
@@ -70,10 +118,8 @@ def search(self, max_results=7):
         search_results = []
 
         # Normalize the results to match the format of the other search APIs
+        # Excluded sites should already be filtered out by the query parameters
         for result in results:
-            # skip youtube results
-            if "youtube.com" in result["link"]:
-                continue
             search_result = {
                 "title": result["title"],
                 "href": result["link"],