Added avatar, namecard and player title in players search endpoint (#82)

.env.dist

@@ -20,7 +20,7 @@ HERO_PATH_CACHE_TIMEOUT=86400
 CSV_CACHE_TIMEOUT=86400
 CAREER_PATH_CACHE_TIMEOUT=7200
 SEARCH_ACCOUNT_PATH_CACHE_TIMEOUT=3600
-NAMECARDS_TIMEOUT=7200
+SEARCH_DATA_TIMEOUT=7200
 CAREER_PARSER_CACHE_EXPIRATION_TIMEOUT=604800
 PARSER_CACHE_EXPIRATION_SPREADING_PERCENTAGE=25
 

app/commands/check_and_update_cache.py

@@ -16,11 +16,11 @@
 from app.parsers.heroes_parser import HeroesParser
 from app.parsers.heroes_stats_parser import HeroesStatsParser
 from app.parsers.maps_parser import MapsParser
-from app.parsers.namecard_parser import NamecardParser
 from app.parsers.player_career_parser import PlayerCareerParser
 from app.parsers.player_parser import PlayerParser
 from app.parsers.player_stats_summary_parser import PlayerStatsSummaryParser
 from app.parsers.roles_parser import RolesParser
+from app.parsers.search_data_parser import NamecardParser
 
 # Mapping of parser class names to linked classes
 PARSER_CLASSES_MAPPING = {

app/commands/update_search_data_cache.py

@@ -0,0 +1,116 @@
+"""Command used in order to retrieve the last version of the namecards"""
+import json
+import re
+
+import httpx
+
+from app.common.cache_manager import CacheManager
+from app.common.enums import Locale, SearchDataType
+from app.common.exceptions import SearchDataRetrievalError
+from app.common.helpers import send_discord_webhook_message
+from app.common.logging import logger
+from app.config import settings
+
+# Generic cache manager used in the process
+cache_manager = CacheManager()
+
+# Mapping between the search data type and the variable name in JS
+variable_name_mapping: dict[SearchDataType, str] = {
+    SearchDataType.PORTRAIT: "avatars",
+    SearchDataType.NAMECARD: "namecards",
+    SearchDataType.TITLE: "titles",
+}
+
+
+def get_search_page() -> httpx.Response:
+    try:
+        response = httpx.get(
+            f"{settings.blizzard_host}/{Locale.ENGLISH_US}{settings.search_data_path}"
+        )
+    except httpx.RequestError as error:
+        logger.exception("An error occurred while requesting search data !")
+        raise SearchDataRetrievalError from error
+    else:
+        return response
+
+
+def extract_search_data(html_content: str, data_type: SearchDataType) -> dict:
+    variable_name = variable_name_mapping[data_type]
+    data_regexp = r"const %s = (\{.*\})\n" % variable_name
+
+    result = re.search(data_regexp, html_content)
+
+    if not result:
+        error_message = f"{data_type} data not found on Blizzard page !"
+        logger.exception(error_message)
+        send_discord_webhook_message(error_message)
+        raise SearchDataRetrievalError
+
+    try:
+        json_result = json.loads(result[1])
+    except ValueError as error:
+        error_message = f"Invalid format for {data_type} data on Blizzard page !"
+        logger.exception(error_message)
+        send_discord_webhook_message(error_message)
+        raise SearchDataRetrievalError from error
+    else:
+        return json_result
+
+
+def transform_search_data(
+    search_data: dict, data_type: SearchDataType
+) -> dict[str, str]:
+    def get_data_type_value(data_value: dict) -> str:
+        match data_type:
+            case SearchDataType.PORTRAIT | SearchDataType.NAMECARD:
+                return data_value["icon"]
+
+            case SearchDataType.TITLE:
+                return data_value["name"]["en_US"]
+
+    return {
+        data_key: get_data_type_value(data_value)
+        for data_key, data_value in search_data.items()
+    }
+
+
+def retrieve_search_data(
+    data_type: SearchDataType, search_page: httpx.Response | None = None
+) -> dict[str, str]:
+    if not search_page:
+        logger.info("Retrieving Blizzard search page...")
+        search_page = get_search_page()
+
+    logger.info("Extracting {} data from HTML data...", data_type)
+    search_data = extract_search_data(search_page.text, data_type)
+
+    logger.info("Transforming data...")
+    return transform_search_data(search_data, data_type)
+
+
+def update_search_data_cache():
+    """Main method of the script"""
+    try:
+        logger.info("Retrieving Blizzard search page...")
+        search_page = get_search_page()
+
+        logger.info("Retrieving search data...")
+        search_data = {
+            data_type: retrieve_search_data(data_type, search_page)
+            for data_type in SearchDataType
+        }
+    except SearchDataRetrievalError as error:
+        raise SystemExit from error
+
+    logger.info("Saving search data...")
+    cache_manager.update_search_data_cache(search_data)
+
+
+def main():
+    """Main method of the script"""
+    update_search_data_cache()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    logger = logger.patch(lambda record: record.update(name="update_search_data_cache"))
+    main()

app/commands/update_test_fixtures.py

@@ -82,8 +82,9 @@ def list_routes_to_update(args: argparse.Namespace) -> dict[str, str]:
         )
 
     if args.home:
-        logger.info("Adding home route...")
+        logger.info("Adding home routes...")
         route_file_mapping[settings.home_path] = "/home.html"
+        route_file_mapping[settings.search_data_path] = "/search.html"
 
     return route_file_mapping
 

app/common/cache_manager.py

@@ -30,6 +30,7 @@
 import redis
 from fastapi import Request
 
+from app.common.enums import SearchDataType
 from app.config import settings
 
 from .helpers import compress_json_value, decompress_json_value, get_spread_value
@@ -158,20 +159,25 @@ def get_soon_expired_cache_keys(self, cache_key_prefix: str) -> Iterator[str]:
             yield key.decode("utf-8").removeprefix(prefix_to_remove)
 
     @redis_connection_handler
-    def update_namecards_cache(self, namecards: dict[str, str]) -> None:
-        for namecard_key, namecard_url in namecards.items():
-            self.redis_server.set(
-                f"{settings.namecard_cache_key_prefix}:{namecard_key}",
-                value=namecard_url,
-                ex=settings.namecards_timeout,
-            )
+    def update_search_data_cache(
+        self, search_data: dict[SearchDataType, dict[str, str]]
+    ) -> None:
+        for data_type, data in search_data.items():
+            for data_key, data_value in data.items():
+                self.redis_server.set(
+                    f"{settings.search_data_cache_key_prefix}:{data_type}:{data_key}",
+                    value=data_value,
+                    ex=settings.search_data_timeout,
+                )
 
     @redis_connection_handler
-    def get_namecard_cache(self, cache_key: str) -> str | None:
-        namecard_cache = self.redis_server.get(
-            f"{settings.namecard_cache_key_prefix}:{cache_key}",
+    def get_search_data_cache(
+        self, data_type: SearchDataType, cache_key: str
+    ) -> str | None:
+        data_cache = self.redis_server.get(
+            f"{settings.search_data_cache_key_prefix}:{data_type}:{cache_key}",
         )
-        return namecard_cache.decode("utf-8") if namecard_cache else None
+        return data_cache.decode("utf-8") if data_cache else None
 
     @redis_connection_handler
     def update_parser_cache_last_update(self, cache_key: str, expire: int) -> None:

app/common/enums.py

@@ -142,3 +142,9 @@ class Locale(StrEnum):
     },
 )
 MapGamemode.__doc__ = "Maps gamemodes keys"
+
+
+class SearchDataType(StrEnum):
+    NAMECARD = "namecard"
+    PORTRAIT = "portrait"
+    TITLE = "title"

app/common/exceptions.py

@@ -2,10 +2,10 @@
 from fastapi import status
 
 
-class NamecardsRetrievalError(Exception):
-    """Generic namecards retrieval Exception"""
+class SearchDataRetrievalError(Exception):
+    """Generic search data retrieval Exception (namecards, titles, etc.)"""
 
-    message = "Error while retrieving namecards"
+    message = "Error while retrieving search data"
 
 
 class OverfastError(Exception):

app/common/helpers.py

@@ -222,3 +222,11 @@ def dict_insert_value_before_key(
 def key_to_label(key: str) -> str:
     """Transform a given key in lowercase format into a human format"""
     return " ".join(s.capitalize() for s in key.split("_"))
+
+
+@cache
+def get_player_title(title: str | None) -> str | None:
+    """Get player title from string extracted from Blizzard page. This is
+    where we're handling the special "no title" case for which we return None
+    """
+    return None if title and title.lower() == "no title" else title

app/config.py

@@ -110,17 +110,17 @@ class Settings(BaseSettings):
     parser_cache_expiration_spreading_percentage: int = 25
 
     ############
-    # NAMECARDS
+    # SEARCH DATA (AVATARS, NAMECARDS, TITLES)
     ############
 
-    # Cache key for namecard cache in Redis.
-    namecard_cache_key_prefix: str = "namecard-cache"
+    # Cache key for search data cache in Redis.
+    search_data_cache_key_prefix: str = "search-data-cache"
 
-    # URI of the page where namecards are saved
-    namecards_path: str = "/en-us/search/"
+    # URI of the page where search data are saved
+    search_data_path: str = "/search/"
 
-    # Cache TTL for namecards list
-    namecards_timeout: int = 7200
+    # Cache TTL for search data list
+    search_data_timeout: int = 7200
 
     ############
     # BLIZZARD

app/handlers/get_player_career_request_handler.py

@@ -2,8 +2,8 @@
 from typing import ClassVar
 
 from app.config import settings
-from app.parsers.namecard_parser import NamecardParser
 from app.parsers.player_parser import PlayerParser
+from app.parsers.search_data_parser import NamecardParser
 
 from .api_request_handler import APIRequestHandler