Rutube videos (#2400)

src/apps/notion/admin/__init__.py

@@ -0,0 +1,9 @@
+from apps.notion.admin.file import NotionMaterialFileAdmin
+from apps.notion.admin.material import NotionMaterialAdmin
+from apps.notion.admin.video import NotionVideoAdmin
+
+__all__ = [
+    "NotionMaterialAdmin",
+    "NotionMaterialFileAdmin",
+    "NotionVideoAdmin",
+]

src/apps/notion/admin/file.py

@@ -0,0 +1,6 @@
+from apps.notion.models import MaterialFile
+from core.admin import ModelAdmin, admin
+
+
+@admin.register(MaterialFile)
+class NotionMaterialFileAdmin(ModelAdmin): ...

src/apps/notion/admin.py → src/apps/notion/admin/material.py

@@ -9,7 +9,7 @@
 from apps.notion import helpers
 from apps.notion.client import NotionClient
 from apps.notion.exceptions import NotionError
-from apps.notion.models import Material, MaterialFile
+from apps.notion.models import Material
 from core.admin import ModelAdmin, admin
 
 
@@ -87,7 +87,3 @@ def notion_page(self, obj: Material) -> str:
         return f"""<a target="_blank" href="{ notion_url }">
             <img class="notion-logo" src="/static/logo/notion.svg" />
             {obj.page_id}</a>"""
-
-
-@admin.register(MaterialFile)
-class MaterialFileAdmin(ModelAdmin): ...

src/apps/notion/admin/video.py

@@ -0,0 +1,55 @@
+from typing import no_type_check
+
+from django import forms
+from django.utils.safestring import mark_safe
+from django.utils.translation import gettext_lazy as _
+
+from apps.notion import helpers
+from apps.notion.models import Video
+from core.admin import ModelAdmin, admin
+
+
+class NotionVideoForm(forms.ModelForm):
+    class Meta:
+        model = Video
+        fields = "__all__"
+
+    @no_type_check
+    def clean_rutube_id(self) -> str:
+        return helpers.get_rutube_video_id(self.cleaned_data["rutube_id"])
+
+    @no_type_check
+    def clean_youtube_id(self) -> str:
+        return helpers.get_youtube_video_id(self.cleaned_data["youtube_id"])
+
+
+@admin.register(Video)
+class NotionVideoAdmin(ModelAdmin):
+    form = NotionVideoForm
+
+    list_display = [
+        "id",
+        "youtube",
+        "rutube",
+    ]
+    fields = [
+        "youtube_id",
+        "rutube_id",
+    ]
+
+    @admin.display(description=_("Youtube"))
+    @mark_safe
+    def youtube(self, obj: Video) -> str:
+        return f"""<a target="_blank" href="{ obj.get_youtube_url() }">
+            <img class="notion-youtube-logo" src="/static/logo/youtube.png" />
+            {obj.youtube_id}</a>"""
+
+    @admin.display(description=_("RuTube"))
+    @mark_safe
+    def rutube(self, obj: Video) -> str:
+        if obj.rutube_id is not None:
+            return f"""<a target="_blank" href="{ obj.get_rutube_url() }">
+                <img class="notion-rutube-logo" src="/static/logo/rutube.png" />
+                {obj.rutube_id}</a>"""
+
+        return "—"

src/apps/notion/helpers.py

@@ -1,5 +1,5 @@
 from os.path import basename
-from urllib.parse import urlparse
+from urllib.parse import parse_qs, urlparse
 
 from apps.notion.types import BlockId
 
@@ -24,3 +24,31 @@ def id_to_uuid(id: BlockId) -> BlockId:
 
 def uuid_to_id(uuid: BlockId) -> BlockId:
     return uuid.replace("-", "")
+
+
+def get_youtube_video_id(url: str) -> str | None:
+    parsed = urlparse(url)
+    if parsed.netloc == "":  # assume non-urls are direct youtube ids
+        return url
+
+    if parsed.netloc not in ["youtu.be", "www.youtube.com"]:
+        return None
+
+    if parsed.query is not None:
+        query_string = parse_qs(parsed.query)
+        if "v" in query_string:
+            return query_string["v"][0]
+
+    return parsed.path.replace("/", "")
+
+
+def get_rutube_video_id(url: str) -> str | None:
+    parsed = urlparse(url)
+
+    if parsed.netloc == "":  # assume non-urls are direct rutube ids
+        return url
+
+    if "rutube" not in parsed.netloc:
+        return None
+
+    return parsed.path.split("/")[-2]

src/apps/notion/migrations/0010_notion_video.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.15 on 2024-09-04 07:38
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("notion", "0009_notion_slug_field_translation"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Video",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created", models.DateTimeField(auto_now_add=True, db_index=True)),
+                ("modified", models.DateTimeField(blank=True, db_index=True, null=True)),
+                ("youtube_id", models.CharField(db_index=True, max_length=32, unique=True)),
+                ("rutube_id", models.CharField(blank=True, db_index=True, max_length=32, null=True)),
+            ],
+            options={
+                "verbose_name": "Notion video",
+                "verbose_name_plural": "Notion videos",
+            },
+            bases=(core.models.TestUtilsMixin, models.Model),
+        ),
+    ]

src/apps/notion/migrations/0011_notion_video_fields_and_description.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.15 on 2024-09-04 18:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("notion", "0010_notion_video"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="video",
+            name="rutube_id",
+            field=models.CharField(blank=True, db_index=True, help_text="Paste it from the address bar", max_length=256, null=True),
+        ),
+        migrations.AlterField(
+            model_name="video",
+            name="youtube_id",
+            field=models.CharField(db_index=True, help_text="Paste it from the address bar", max_length=256, unique=True),
+        ),
+    ]

src/apps/notion/models/__init__.py

@@ -2,10 +2,12 @@
 from apps.notion.models.cache_entry import NotionCacheEntry
 from apps.notion.models.material import Material
 from apps.notion.models.material_file import MaterialFile
+from apps.notion.models.video import Video
 
 __all__ = [
     "NotionAsset",
     "NotionCacheEntry",
     "Material",
     "MaterialFile",
+    "Video",
 ]

src/apps/notion/models/video.py

@@ -0,0 +1,29 @@
+from django.utils.translation import gettext_lazy as _
+
+from core.models import TimestampedModel, models
+
+
+class Video(TimestampedModel):
+    """Video mapping for multiple videohostings"""
+
+    youtube_id = models.CharField(max_length=256, unique=True, db_index=True, help_text=_("Paste it from the address bar"))
+    rutube_id = models.CharField(max_length=256, blank=True, null=True, db_index=True, help_text=_("Paste it from the address bar"))
+
+    class Meta:
+        verbose_name = _("Notion video")
+        verbose_name_plural = _("Notion videos")
+
+    def __str__(self) -> str:
+        return self.youtube_id
+
+    def get_youtube_embed_src(self) -> str:
+        return f"https://www.youtube.com/embed/{ self.youtube_id }?rel=0"
+
+    def get_youtube_url(self) -> str:
+        return f"https://youtu.be/{ self.youtube_id }"
+
+    def get_rutube_embed_src(self) -> str:
+        return f"https://rutube.ru/play/embed/{self.rutube_id }/"
+
+    def get_rutube_url(self) -> str:
+        return f"https://rutube.ru/video/{ self.rutube_id }/"

src/apps/notion/rewrite/__init__.py

@@ -2,13 +2,21 @@
 from apps.notion.rewrite.links import rewrite_links
 from apps.notion.rewrite.notion_so_assets import rewrite_notion_so_assets
 from apps.notion.rewrite.tags import drop_extra_tags
+from apps.notion.rewrite.video import rewrite_video
 from apps.notion.types import BlockData
+from core.request import get_request
 
 
 def apply_our_adjustments(notion_block_data: BlockData) -> BlockData:
     adjusted = rewrite_links(notion_block_data)  # replace material ids
     adjusted = drop_extra_tags(adjusted)  # remove tags not needed in frontend rendering
     adjusted = rewrite_fetched_assets(adjusted)  # replace asset links with links to our cdn
+
+    request = get_request()
+    if request is not None and request.country_code == "RU":
+        if "frkn" in request.headers:
+            adjusted = rewrite_video(adjusted)  # apply video adjustements for russian users
+
     return rewrite_notion_so_assets(adjusted)  # rewrite remaining assets to go though notion.so so they would sign our request to their S3
 
 
@@ -18,4 +26,5 @@ def apply_our_adjustments(notion_block_data: BlockData) -> BlockData:
     "rewrite_fetched_assets",
     "rewrite_links",
     "rewrite_notion_so_assets",
+    "rewrite_video",
 ]

src/apps/notion/rewrite/video.py

@@ -0,0 +1,49 @@
+from typing import Mapping
+
+from django.core.cache import cache
+
+from apps.notion.helpers import get_youtube_video_id
+from apps.notion.models import Video
+from apps.notion.types import BlockData as NotionBlockData
+
+
+def get_video_mapping() -> Mapping[str, Mapping[str, str]]:
+    cached = cache.get("notion-video-mapping")
+    if cached is not None:
+        return cached
+
+    mapping = dict()
+
+    for video in Video.objects.all().iterator():
+        mapping[video.youtube_id] = {
+            "youtube_embed": video.get_youtube_embed_src(),
+            "rutube_embed": video.get_rutube_embed_src(),
+            "rutube_url": video.get_rutube_url(),
+            "youtube_url": video.get_youtube_url(),
+        }
+
+    cache.set(key="notion-video-mapping", value=mapping, timeout=60)
+
+    return mapping
+
+
+def rewrite_video(block_data: NotionBlockData) -> NotionBlockData:
+    if "type" not in block_data.get("value", {}):  # skip rewrite for untyped blocks
+        return block_data
+
+    if block_data["value"]["type"] != "video":
+        return block_data
+
+    video_id = get_youtube_video_id(block_data["value"]["properties"]["source"][0][0])
+
+    if video_id is None:  # skip rewrite for non-youtube videos
+        return block_data
+
+    video_mapping = get_video_mapping()
+
+    if video_id in video_mapping:
+        block_data["value"]["properties"]["source"] = [[video_mapping[video_id]["rutube_url"]]]
+        block_data["value"]["format"]["link_provider"] = "RuTube"
+        block_data["value"]["format"]["display_source"] = video_mapping[video_id]["rutube_embed"]
+
+    return block_data

src/apps/notion/tests/notion/api/conftest.py

@@ -7,6 +7,11 @@
 ]
 
 
+@pytest.fixture
+def disable_notion_cache(mocker):
+    return mocker.patch("apps.notion.cache.should_bypass_cache", return_value=True)
+
+
 @pytest.fixture(autouse=True)
 def set_current_user(_set_current_user):
     return _set_current_user

src/apps/notion/tests/notion/api/test_notion_api_basic.py

@@ -0,0 +1,45 @@
+import pytest
+
+pytestmark = [
+    pytest.mark.django_db,
+    pytest.mark.single_thread,
+    pytest.mark.usefixtures(
+        "mock_notion_response",
+        "_cdn_dev_storage",
+    ),
+]
+
+
+@pytest.mark.parametrize("material_id", ["0e5693d2-173a-4f77-ae81-06813b6e5329", "0e5693d2173a4f77ae8106813b6e5329"])
+def test_both_formats_work_with_id(api, material_id, mock_notion_response):
+    api.get(f"/api/v2/notion/materials/{material_id}/")
+
+    mock_notion_response.assert_called_once_with("0e5693d2173a4f77ae8106813b6e5329")
+
+
+@pytest.mark.parametrize("material_slug", ["4d5726e8-ee52-4448-b8f9-7be4c7f8e632", "4d5726e8ee524448b8f97be4c7f8e632"])
+def test_both_formats_work_with_slug(api, material_slug, mock_notion_response):
+    api.get(f"/api/v2/notion/materials/{material_slug}/")
+
+    mock_notion_response.assert_called_once_with("0e5693d2173a4f77ae8106813b6e5329")  # original material id
+
+
+def test_content_is_passed_from_notion_client(api, material):
+    got = api.get(f"/api/v2/notion/materials/{material.page_id}/")
+
+    assert got["block-1"]["value"]["parent_id"] == "100500"
+    assert got["block-2"]["value"]["parent_id"] == "100600"
+
+
+def test_404_for_non_existant_materials(api, mock_notion_response):
+    api.get("/api/v2/notion/materials/nonexistant/", expected_status_code=404)
+
+    mock_notion_response.assert_not_called()
+
+
+def test_404_for_inactive_materials(api, mock_notion_response, material):
+    material.update(active=False)
+
+    api.get(f"/api/v2/notion/materials/{material.page_id}/", expected_status_code=404)
+
+    mock_notion_response.assert_not_called()