support BoardGameGeek

boofilsic/settings.py

@@ -498,7 +498,7 @@
 
 SEARCH_INDEX_NEW_ONLY = False
 
-DOWNLOADER_SAVEDIR = env("NEODB_DOWNLOADER_SAVE_DIR", default=None)  # type: ignore
+DOWNLOADER_SAVEDIR = env("NEODB_DOWNLOADER_SAVE_DIR", default="/tmp")  # type: ignore
 
 DISABLE_MODEL_SIGNAL = False  # disable index and social feeds during importing/etc
 

catalog/common/downloaders.py

@@ -11,7 +11,7 @@
 import requests
 from django.conf import settings
 from django.core.cache import cache
-from lxml import html
+from lxml import etree, html
 from PIL import Image
 from requests import Response
 from requests.exceptions import RequestException
@@ -85,6 +85,9 @@ def html(self):
             self.content.decode("utf-8")
         )
 
+    def xml(self):
+        return etree.fromstring(self.content, base_url=self.url)
+
     @property
     def headers(self):
         return {
@@ -93,6 +96,7 @@ def headers(self):
 
 
 requests.Response.html = MockResponse.html  # type:ignore
+requests.Response.xml = MockResponse.xml  # type:ignore
 
 
 class DownloaderResponse(Response):
@@ -101,6 +105,9 @@ def html(self):
             self.content.decode("utf-8")
         )
 
+    def xml(self):
+        return etree.fromstring(self.content, base_url=self.url)
+
 
 class DownloadError(Exception):
     def __init__(self, downloader, msg=None):

catalog/common/models.py

@@ -42,6 +42,7 @@ class SiteName(models.TextChoices):
     IGDB = "igdb", _("IGDB")
     Steam = "steam", _("Steam")
     Bangumi = "bangumi", _("Bangumi")
+    BGG = "bgg", _("BGG")
     # ApplePodcast = "apple_podcast", _("苹果播客")
     RSS = "rss", _("RSS")
     Discogs = "discogs", _("Discogs")
@@ -87,6 +88,7 @@ class IdType(models.TextChoices):
     Spotify_Artist = "spotify_artist", _("Spotify艺术家")
     TMDB_Person = "tmdb_person", _("TMDB影人")
     IGDB = "igdb", _("IGDB游戏")
+    BGG = "bgg", _("BGG桌游")
     Steam = "steam", _("Steam游戏")
     Bangumi = "bangumi", _("Bangumi")
     ApplePodcast = "apple_podcast", _("苹果播客")

catalog/common/sites.py

@@ -104,6 +104,10 @@ def scrape_additional_data(self):
     def query_str(content, query: str) -> str:
         return content.xpath(query)[0].strip()
 
+    @staticmethod
+    def query_list(content, query: str) -> list[str]:
+        return list(content.xpath(query))
+
     @classmethod
     def match_existing_item_for_resource(
         cls, resource: ExternalResource

catalog/game/models.py

@@ -43,8 +43,11 @@ class Game(Item):
         "title",
         "brief",
         "other_title",
+        "designer",
+        "artist",
         "developer",
         "publisher",
+        "release_year",
         "release_date",
         "genre",
         "platform",
@@ -59,6 +62,22 @@ class Game(Item):
         default=list,
     )
 
+    designer = jsondata.ArrayField(
+        base_field=models.CharField(blank=True, default="", max_length=500),
+        verbose_name=_("设计者"),
+        null=True,
+        blank=True,
+        default=list,
+    )
+
+    artist = jsondata.ArrayField(
+        base_field=models.CharField(blank=True, default="", max_length=500),
+        verbose_name=_("艺术家"),
+        null=True,
+        blank=True,
+        default=list,
+    )
+
     developer = jsondata.ArrayField(
         base_field=models.CharField(blank=True, default="", max_length=500),
         verbose_name=_("开发商"),
@@ -75,6 +94,8 @@ class Game(Item):
         default=list,
     )
 
+    release_year = jsondata.IntegerField(verbose_name=_("发布年份"), null=True, blank=True)
+
     release_date = jsondata.DateField(
         verbose_name=_("发布日期"),
         auto_now=False,
@@ -106,6 +127,7 @@ def lookup_id_type_choices(cls):
         id_types = [
             IdType.IGDB,
             IdType.Steam,
+            IdType.BGG,
             IdType.DoubanGame,
             IdType.Bangumi,
         ]

catalog/game/tests.py

@@ -107,6 +107,7 @@ def test_scrape(self):
 
 
 class BangumiGameTestCase(TestCase):
+    @use_local_response
     def test_parse(self):
         t_id_type = IdType.Bangumi
         t_id_value = "15912"
@@ -118,10 +119,24 @@ def test_parse(self):
         self.assertEqual(site.url, t_url)
         self.assertEqual(site.id_value, t_id_value)
 
+
+class BoardGameGeekTestCase(TestCase):
     @use_local_response
     def test_scrape(self):
-        # TODO
-        pass
+        t_url = "https://boardgamegeek.com/boardgame/167791"
+        site = SiteManager.get_site_by_url(t_url)
+        self.assertIsNotNone(site)
+        self.assertEqual(site.ID_TYPE, IdType.BGG)
+        self.assertEqual(site.id_value, "167791")
+        self.assertEqual(site.ready, False)
+        site.get_resource_ready()
+        self.assertEqual(site.ready, True)
+        self.assertEqual(site.resource.metadata["title"], "Terraforming Mars")
+        self.assertIsInstance(site.resource.item, Game)
+        self.assertEqual(site.resource.item.platform, ["Boardgame"])
+        self.assertEqual(site.resource.item.genre[0], "Economic")
+        self.assertEqual(site.resource.item.other_title[0], "殖民火星")
+        self.assertEqual(site.resource.item.designer, ["Jacob Fryxelius"])
 
 
 class MultiGameSitesTestCase(TestCase):

catalog/sites/__init__.py

@@ -2,6 +2,7 @@
 from .apple_music import AppleMusic
 from .bandcamp import Bandcamp
 from .bangumi import Bangumi
+from .bgg import BoardGameGeek
 from .bookstw import BooksTW
 from .discogs import DiscogsMaster, DiscogsRelease
 from .douban_book import DoubanBook

catalog/sites/bgg.py

@@ -0,0 +1,87 @@
+"""
+BoardGameGeek
+
+ref: https://boardgamegeek.com/wiki/page/BGG_XML_API2
+"""
+import html
+
+from langdetect import detect
+from loguru import logger
+
+from catalog.common import *
+from catalog.models import *
+
+
+def _lang(s: str) -> str:
+    try:
+        return detect(s)
+    except Exception:
+        return "en"
+
+
+@SiteManager.register
+class BoardGameGeek(AbstractSite):
+    SITE_NAME = SiteName.BGG
+    ID_TYPE = IdType.BGG
+    URL_PATTERNS = [
+        r"^\w+://boardgamegeek\.com/boardgame/(\d+)",
+    ]
+    WIKI_PROPERTY_ID = "?"
+    DEFAULT_MODEL = Game
+
+    @classmethod
+    def id_to_url(cls, id_value):
+        return "https://boardgamegeek.com/boardgame/" + id_value
+
+    def scrape(self):
+        api_url = f"https://boardgamegeek.com/xmlapi2/thing?stats=1&type=boardgame&id={self.id_value}"
+        content = BasicDownloader(api_url).download().xml()
+        items = list(content.xpath("/items/item"))  # type: ignore
+        if not len(items):
+            raise ParseError("boardgame not found", field="id")
+        item = items[0]
+        title = self.query_str(item, "name[@type='primary']/@value")
+        other_title = self.query_list(item, "name[@type='alternate']/@value")
+        zh_title = [
+            t for t in other_title if _lang(t) in ["zh", "jp", "ko", "zh-cn", "zh-tw"]
+        ]
+        if zh_title:
+            for z in zh_title:
+                other_title.remove(z)
+            other_title = zh_title + other_title
+
+        cover_image_url = self.query_str(item, "image/text()")
+        brief = html.unescape(self.query_str(item, "description/text()"))
+        year = self.query_str(item, "yearpublished/@value")
+        designer = self.query_list(item, "link[@type='boardgamedesigner']/@value")
+        artist = self.query_list(item, "link[@type='boardgameartist']/@value")
+        publisher = self.query_list(item, "link[@type='boardgamepublisher']/@value")
+        developer = self.query_list(item, "link[@type='boardgamedeveloper']/@value")
+        category = self.query_list(item, "link[@type='boardgamecategory']/@value")
+
+        pd = ResourceContent(
+            metadata={
+                "title": title,
+                "other_title": other_title,
+                "genre": category,
+                "developer": developer,
+                "publisher": publisher,
+                "designer": designer,
+                "artist": artist,
+                "release_year": year,
+                "platform": ["Boardgame"],
+                "brief": brief,
+                # "official_site": official_site,
+                "cover_image_url": cover_image_url,
+            }
+        )
+        if pd.metadata["cover_image_url"]:
+            imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url)
+            try:
+                pd.cover_image = imgdl.download().content
+                pd.cover_image_extention = imgdl.extention
+            except Exception:
+                logger.debug(
+                    f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}'
+                )
+        return pd

catalog/templates/_people.html

@@ -3,7 +3,7 @@
     {% if role %}{{ role }}：{% endif %}
     {% for p in people %}
       {% if forloop.counter <= max %}
-        {% if not forloop.first %}、{% endif %}
+        {% if not forloop.first %}/{% endif %}
         <span>{{ p }}</span>
       {% elif forloop.last %}
         等

catalog/templates/game.html

@@ -11,73 +11,25 @@
 {% load thumb %}
 <!-- class specific details -->
 {% block details %}
-  <div>
-    {% if item.other_title %}
-      {% trans '别名：' %}
-      {% for other_title in item.other_title %}
-        <span {% if forloop.counter > 5 %}style="display: none;"{% endif %}>
-          <span class="other_title">{{ other_title }}</span>
-          {% if not forloop.last %}/{% endif %}
-        </span>
-      {% endfor %}
-      {% if item.other_title|length > 5 %}
-        <a href="javascript:void(0);" id="otherTitleMore">{% trans '更多' %}</a>
-        <script>
-			$("#otherTitleMore").on('click', function (e) {
-				$("span.other_title:not(:visible)").each(function (e) {
-					$(this).parent().removeAttr('style');
-				});
-				$(this).remove();
-			})
-
-        </script>
-      {% endif %}
-    {% endif %}
-  </div>
-  <div>
-    {% if item.genre %}
-      {% trans '类型：' %}
-      {% for genre in item.genre %}
-        <span>{{ genre }}</span>
-        {% if not forloop.last %}/{% endif %}
-      {% endfor %}
-    {% endif %}
-  </div>
-  <div>
-    {% if item.developer %}
-      {% trans '开发商：' %}
-      {% for developer in item.developer %}
-        <span>{{ developer }}</span>
-        {% if not forloop.last %}/{% endif %}
-      {% endfor %}
-    {% endif %}
-  </div>
-  <div>
-    {% if item.publisher %}
-      {% trans '发行商：' %}
-      {% for publisher in item.publisher %}
-        <span>{{ publisher }}</span>
-        {% if not forloop.last %}/{% endif %}
-      {% endfor %}
-    {% endif %}
+  <div class="tldr-2" _="on click toggle .tldr-2 on me">
+    {% include '_people.html' with people=item.other_title _role='别名' max=99 %}
   </div>
   <div>
     {% if item.release_date %}
-      {% trans '发行日期：' %}{{ item.release_date }}
+      {% trans '发行时间：' %}{{ item.release_date }}
+    {% elif item.release_year %}
+      {% trans '发行时间：' %}{{ item.release_year }}
     {% endif %}
   </div>
+  <div>{% include '_people.html' with people=item.platform role='平台' max=8 %}</div>
+  <div>{% include '_people.html' with people=item.genre role='类型' max=5 %}</div>
+  <div>{% include '_people.html' with people=item.designer role='设计者' max=3 %}</div>
+  <div>{% include '_people.html' with people=item.artist role='艺术家' max=3 %}</div>
+  <div>{% include '_people.html' with people=item.developer role='开发商' max=1 %}</div>
+  <div>{% include '_people.html' with people=item.publisher role='发行商' max=1 %}</div>
   <div>
     {% if item.official_site %}
       {% trans '官方网站：' %}{{ item.official_site|urlizetrunc:24 }}
     {% endif %}
   </div>
-  <div>
-    {% if item.platform %}
-      {% trans '平台：' %}
-      {% for platform in item.platform %}
-        <span>{{ platform }}</span>
-        {% if not forloop.last %}/{% endif %}
-      {% endfor %}
-    {% endif %}
-  </div>
 {% endblock %}

common/static/scss/_sitelabel.scss

@@ -45,6 +45,14 @@
     font-weight: bold;
   }
 
+  .bgg {
+    background-color: #3F3A60;
+    color: #FFFFFF;
+    font-weight: bold;
+    //#FC3808;
+    border: none;
+  }
+
   .steam {
     background: linear-gradient(30deg, #1387b8, #111d2e);
     color: white;

compose.yml

@@ -112,7 +112,7 @@ services:
       - ${NEODB_DATA:-../data}/redis:/data
 
   typesense:
-    image: typesense/typesense:0.25.1
+    image: typesense/typesense:0.25.2
     restart: "on-failure"
     # healthcheck:
     #   test: ['CMD', 'curl', '-vf', 'http://127.0.0.1:8108/health']