Skip to content

Commit

Permalink
fixes #337, ensure media ids are str in imports
Browse files Browse the repository at this point in the history
  • Loading branch information
FuzzyGrim committed Feb 8, 2025
1 parent d2acb4e commit 599af6c
Show file tree
Hide file tree
Showing 8 changed files with 45 additions and 22 deletions.
9 changes: 9 additions & 0 deletions src/integrations/helpers.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import datetime
import json
import logging

from django.contrib import messages
from django.utils import timezone
Expand All @@ -8,6 +9,8 @@

import app

logger = logging.getLogger(__name__)


def update_season_references(seasons, user):
"""Update season references with actual TV instances.
Expand All @@ -31,6 +34,7 @@ def update_season_references(seasons, user):
media_id = season.item.media_id
if media_id in existing_tv:
season.related_tv = existing_tv[media_id]
logger.debug("Updated season %s with TV %s", season, existing_tv[media_id])


def update_episode_references(episodes, user):
Expand Down Expand Up @@ -58,6 +62,11 @@ def update_episode_references(episodes, user):
)
if season_key in existing_seasons:
episode.related_season = existing_seasons[season_key]
logger.debug(
"Updated episode %s with season %s",
episode,
existing_seasons[season_key],
)


def bulk_chunk_import(bulk_media, model, user, mode):
Expand Down
4 changes: 3 additions & 1 deletion src/integrations/imports/anilist.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

def importer(username, user, mode):
"""Import anime and manga ratings from Anilist."""
logger.info("Starting AniList import for user %s with mode %s", username, mode)

query = """
query ($userName: String){
anime: MediaListCollection(userName: $userName, type: ANIME) {
Expand Down Expand Up @@ -147,7 +149,7 @@ def process_status_list(bulk_media, status_list, media_type, user, warnings):
notes = content["notes"] or ""

item, _ = app.models.Item.objects.get_or_create(
media_id=content["media"]["idMal"],
media_id=str(content["media"]["idMal"]),
source="mal",
media_type=media_type,
defaults={
Expand Down
6 changes: 4 additions & 2 deletions src/integrations/imports/kitsu.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ def importer(kitsu_id, user, mode):
if not kitsu_id.isdigit():
kitsu_id = get_kitsu_id(kitsu_id)

logger.info("Starting Kitsu import for user id %s with mode %s", kitsu_id, mode)

anime_response = get_media_response(kitsu_id, "anime")
num_anime_imported, anime_warnings = import_media(
anime_response,
Expand Down Expand Up @@ -191,7 +193,7 @@ def create_or_get_item(media_type, kitsu_metadata, mapping_lookup, kitsu_mu_mapp

external_id = mappings[site]
if site == f"myanimelist/{media_type}":
media_id = int(external_id)
media_id = external_id
season_number = None
source = "mal"
break
Expand All @@ -206,7 +208,7 @@ def create_or_get_item(media_type, kitsu_metadata, mapping_lookup, kitsu_mu_mapp
continue

# decode the base36 encoded ID
media_id = int(external_id, 36)
media_id = str(int(external_id, 36))
media_type = "manga"
season_number = None
source = "mangaupdates"
Expand Down
4 changes: 3 additions & 1 deletion src/integrations/imports/mal.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

def importer(username, user, mode):
"""Import anime and manga from MyAnimeList."""
logger.info("Starting MyAnimeList import for user %s with mode %s", username, mode)

anime_imported = import_media(username, user, "anime", mode)
manga_imported = import_media(username, user, "manga", mode)
return anime_imported, manga_imported
Expand Down Expand Up @@ -90,7 +92,7 @@ def add_media_list(response, media_type, user):
image_url = settings.IMG_NONE

item, _ = app.models.Item.objects.get_or_create(
media_id=content["node"]["id"],
media_id=str(content["node"]["id"]),
source="mal",
media_type=media_type,
defaults={
Expand Down
17 changes: 11 additions & 6 deletions src/integrations/imports/simkl.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ def get_token(request):

def importer(token, user, mode):
"""Import tv shows, movies and anime from SIMKL."""
logger.info("Starting SIMKL import with mode %s", mode)

data = get_user_list(token)

if not data:
Expand All @@ -67,12 +69,15 @@ def importer(token, user, mode):
# Import using bulk operations
imported_counts = {}
for media_type, bulk_list in bulk_media.items():
imported_counts[media_type] = helpers.bulk_chunk_import(
bulk_list,
apps.get_model(app_label="app", model_name=media_type),
user,
mode,
)
logger.info("Bulk creating %d %s", len(bulk_list), media_type)

if bulk_list:
imported_counts[media_type] = helpers.bulk_chunk_import(
bulk_list,
apps.get_model(app_label="app", model_name=media_type),
user,
mode,
)

return (
imported_counts.get("tv", 0),
Expand Down
20 changes: 12 additions & 8 deletions src/integrations/imports/trakt.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

def importer(username, user, mode):
"""Import the user's data from Trakt."""
logger.info("Starting Trakt import for user %s with mode %s", username, mode)

user_base_url = f"{TRAKT_API_BASE_URL}/users/{username}"
mal_shows_map = get_mal_mappings(is_show=True)
mal_movies_map = get_mal_mappings(is_show=False)
Expand Down Expand Up @@ -104,6 +106,8 @@ def importer(username, user, mode):
# Bulk create all media types
imported_counts = {}
for media_type, bulk_list in bulk_media.items():
logger.info("Bulk creating %d %s", len(bulk_list), media_type)

if bulk_list: # Only process non-empty lists
imported_counts[media_type] = helpers.bulk_chunk_import(
bulk_list,
Expand Down Expand Up @@ -167,7 +171,7 @@ def process_watched_shows(
media_instances,
)
else:
tmdb_id = entry["show"]["ids"]["tmdb"]
tmdb_id = str(entry["show"]["ids"]["tmdb"])
if not tmdb_id:
warnings.append(
f"No TMDB ID found for {trakt_title} in watch history",
Expand Down Expand Up @@ -369,8 +373,8 @@ def update_or_prepare_show(
):
"""Update existing show or prepare new one for bulk creation."""
trakt_id = entry["show"]["ids"]["trakt"]
tmdb_id = entry["show"]["ids"]["tmdb"]
mal_id = mal_shows_map.get((trakt_id, 1))
tmdb_id = str(entry["show"]["ids"]["tmdb"])
mal_id = str(mal_shows_map.get((trakt_id, 1)))

if mal_id and user.anime_enabled:
if mal_id in media_instances["anime"]:
Expand Down Expand Up @@ -405,7 +409,7 @@ def update_or_prepare_movie(
):
"""Update existing movie or prepare new one for bulk creation."""
trakt_id = entry["movie"]["ids"]["trakt"]
tmdb_id = entry["movie"]["ids"]["tmdb"]
tmdb_id = str(entry["movie"]["ids"]["tmdb"])
mal_id = mal_mapping.get((trakt_id, 1))

if mal_id and user.anime_enabled:
Expand Down Expand Up @@ -448,7 +452,7 @@ def update_or_prepare_season(
):
"""Update existing season or prepare new one for bulk creation."""
trakt_id = entry["show"]["ids"]["trakt"]
tmdb_id = entry["show"]["ids"]["tmdb"]
tmdb_id = str(entry["show"]["ids"]["tmdb"])
season_number = entry["season"]["number"]
mal_id = mal_shows_map.get((trakt_id, season_number))

Expand Down Expand Up @@ -485,7 +489,7 @@ def update_or_prepare_season(

def prepare_tmdb_show(entry, user, defaults, list_type, bulk_media, media_instances):
"""Prepare TMDB show for bulk creation."""
tmdb_id = entry["show"]["ids"]["tmdb"]
tmdb_id = str(entry["show"]["ids"]["tmdb"])
trakt_title = entry["show"]["title"]

if not tmdb_id:
Expand Down Expand Up @@ -585,7 +589,7 @@ def prepare_tmdb_season_and_episodes(

def prepare_tmdb_season(entry, user, defaults, list_type, bulk_media, media_instances):
"""Prepare TMDB season for bulk creation."""
tmdb_id = entry["show"]["ids"]["tmdb"]
tmdb_id = str(entry["show"]["ids"]["tmdb"])
trakt_title = entry["show"]["title"]
season_number = entry["season"]["number"]

Expand Down Expand Up @@ -646,7 +650,7 @@ def prepare_tmdb_season(entry, user, defaults, list_type, bulk_media, media_inst

def prepare_tmdb_movie(entry, user, defaults, list_type, bulk_media, media_instances):
"""Prepare TMDB movie for bulk creation."""
tmdb_id = entry["movie"]["ids"]["tmdb"]
tmdb_id = str(entry["movie"]["ids"]["tmdb"])
trakt_title = entry["movie"]["title"]

if not tmdb_id:
Expand Down
5 changes: 2 additions & 3 deletions src/integrations/imports/yamtrack.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,12 @@

def importer(file, user, mode):
"""Import media from CSV file."""
logger.info("Starting Yamtrack import with mode %s", mode)

decoded_file = file.read().decode("utf-8").splitlines()
reader = DictReader(decoded_file)

logger.info("Importing from Yamtrack")

bulk_media = {media_type: [] for media_type in Item.MediaTypes.values}

imported_counts = {}

for row in reader:
Expand Down
2 changes: 1 addition & 1 deletion src/integrations/tests/test_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ def test_process_entry(self):
self.user,
)

self.assertEqual(instance.item.media_id, 1)
self.assertEqual(instance.item.media_id, "1")
self.assertIsInstance(instance, Anime)
self.assertEqual(instance.score, 9)
self.assertEqual(instance.progress, 26)
Expand Down

0 comments on commit 599af6c

Please sign in to comment.