From ce50bead0b632e2ae95e5acc42b07da48c64893b Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Sat, 14 May 2022 18:28:30 +0200 Subject: [PATCH 01/15] Added volume support --- MangaTaggerLib/MangaTaggerLib.py | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/MangaTaggerLib/MangaTaggerLib.py b/MangaTaggerLib/MangaTaggerLib.py index bbd374a..b3b0a12 100644 --- a/MangaTaggerLib/MangaTaggerLib.py +++ b/MangaTaggerLib/MangaTaggerLib.py @@ -65,7 +65,7 @@ def process_manga_chapter(file_path: Path, event_id): manga_details = filename_parser(filename, logging_info) - metadata_tagger(file_path, manga_details[0], manga_details[1], manga_details[2], logging_info) + metadata_tagger(file_path, manga_details[0], manga_details[1], manga_details[2], logging_info,manga_details[3]) # Remove manga directory if empty try: @@ -159,8 +159,12 @@ def filename_parser(filename, logging_info): LOG.debug(f'chapter_number: {chapter_number}') logging_info['chapter_number'] = chapter_number - - return manga_title, chapter_number, format + volume = re.findall(r"(?:Vol|v|volume)(?:\s|\.)?(?:\s|\.)?([0-9]+(?:\.[0-9]+)?)",chapter_title) + if volume: + volume = volume[0] + else: + volume = None + return manga_title, chapter_number, format, volume def rename_action(current_file_path: Path, new_file_path: Path, manga_title, chapter_number, logging_info): @@ -256,7 +260,7 @@ def compare_versions(old_filename: str, new_filename: str): return False -def metadata_tagger(file_path, manga_title, manga_chapter_number, format, logging_info): +def metadata_tagger(file_path, manga_title, manga_chapter_number, format, logging_info,volume): manga_search = None db_exists = True retries = 0 @@ -318,7 +322,10 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin series_title_legal = slugify(series_title) manga_library_dir = Path(AppSettings.library_dir, series_title_legal) try: - new_filename = f"{series_title_legal} {manga_chapter_number}.cbz" + if volume is not None: + new_filename = f"{series_title_legal} Vol.{volume} {manga_chapter_number}.cbz" + else: + new_filename = f"{series_title_legal} {manga_chapter_number}.cbz" LOG.debug(f'new_filename: {new_filename}') except TypeError: LOG.warning(f'Manga Tagger was unable to process "{file_path}"', extra=logging_info) @@ -383,7 +390,10 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin LOG.debug(f'anilist_details: {anilist_details}') try: - new_filename = f"{series_title_legal} {manga_chapter_number}.cbz" + if volume is not None: + new_filename = f"{series_title_legal} Vol.{volume} {manga_chapter_number}.cbz" + else: + new_filename = f"{series_title_legal} {manga_chapter_number}.cbz" LOG.debug(f'new_filename: {new_filename}') except TypeError: LOG.warning(f'Manga Tagger was unable to process "{file_path}"', extra=logging_info) @@ -451,7 +461,7 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin else: LOG.info('Image Directory not set, not downloading series cover image.', extra=logging_info) - comicinfo_xml = construct_comicinfo_xml(manga_metadata, manga_chapter_number, logging_info) + comicinfo_xml = construct_comicinfo_xml(manga_metadata, manga_chapter_number, logging_info,volume) reconstruct_manga_chapter(series_title, comicinfo_xml, new_file_path, logging_info) LOG.info(f'Processing on "{new_file_path}" has finished.', extra=logging_info) @@ -471,7 +481,7 @@ def construct_anilist_titles(anilist_details): return anilist_titles -def construct_comicinfo_xml(metadata, chapter_number, logging_info): +def construct_comicinfo_xml(metadata, chapter_number, logging_info,volume_number): LOG.info(f'Constructing comicinfo object for "{metadata.series_title}", chapter {chapter_number}...', extra=logging_info) @@ -484,11 +494,16 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info): series.text = metadata.series_title if metadata.series_title_eng is not None and compare(metadata.series_title, metadata.series_title_eng) != 1: - alt_series = SubElement(comicinfo, 'AlternateSeries') - alt_series.text = metadata.series_title_eng + # alt_series = SubElement(comicinfo, 'AlternateSeries') + # alt_series.text = metadata.series_title_eng + # if metadata.series_title_eng is not None: + localized_series = SubElement(comicinfo,'LocalizedSeries') number = SubElement(comicinfo, 'Number') number.text = f'{chapter_number}' + if volume_number is not None: + volume = SubElement(comicinfo, 'Volume') + volume.text = f'{volume_number}' summary = SubElement(comicinfo, 'Summary') soup = BeautifulSoup(metadata.description, "html.parser") From 22a990a18bbd5055b9b306a69c041ca83687eea5 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Sat, 14 May 2022 18:38:44 +0200 Subject: [PATCH 02/15] added volume support to unit test --- tests/test_manga.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_manga.py b/tests/test_manga.py index 58f69c6..e04cf9b 100644 --- a/tests/test_manga.py +++ b/tests/test_manga.py @@ -15,7 +15,7 @@ def test_filename_parser_manga(self): filename = "Naruto -.- Chap 0.cbz" directory_name = "Naruto" logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } - expected_result = ("Naruto", "000", "MANGA") + expected_result = ("Naruto", "000", "MANGA",None) result = filename_parser(filename, logging_info) self.assertEqual(expected_result, result) @@ -23,7 +23,7 @@ def test_filename_parser_decnumber(self): filename = "Naruto -.- Chap 15.5.cbz" directory_name = "Naruto" logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } - expected_result = ("Naruto", "015.5", "MANGA") + expected_result = ("Naruto", "015.5", "MANGA",None) result = filename_parser(filename, logging_info) self.assertEqual(expected_result, result) @@ -32,7 +32,7 @@ def test_filename_parser_oneshot(self): filename = "Naruto -.- Oneshot.cbz" directory_name = "Naruto" logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } - expected_result = ("Naruto", "000", "ONE_SHOT") + expected_result = ("Naruto", "000", "ONE_SHOT",None) result = filename_parser(filename, logging_info) self.assertEqual(expected_result, result) @@ -41,7 +41,7 @@ def test_filename_parser_prologue(self): filename = "Berserk -.- Prologue 5.cbz" directory_name = "Berserk" logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } - expected_result = ("Berserk", "000.5", "MANGA") + expected_result = ("Berserk", "000.5", "MANGA",None) result = filename_parser(filename, logging_info) self.assertEqual(expected_result, result) @@ -50,7 +50,7 @@ def test_filename_parser_ignore_fluff(self): filename = "One Piece -.- Volume 50 Episode 156 A Chapter Name (15).cbz" directory_name = "Naruto" logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } - expected_result = ("One Piece", "156", "MANGA") + expected_result = ("One Piece", "156", "MANGA","50") result = filename_parser(filename, logging_info) self.assertEqual(expected_result, result) From 624182430239cf3147b5735589304460cec0b99c Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 16 May 2022 00:03:37 +0200 Subject: [PATCH 03/15] Parse volume earlier since parsing chapter number removes anything that is not chapter number. --- MangaTaggerLib/MangaTaggerLib.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/MangaTaggerLib/MangaTaggerLib.py b/MangaTaggerLib/MangaTaggerLib.py index b3b0a12..ec6c69a 100644 --- a/MangaTaggerLib/MangaTaggerLib.py +++ b/MangaTaggerLib/MangaTaggerLib.py @@ -96,7 +96,11 @@ def filename_parser(filename, logging_info): LOG.debug(f'chapter: {chapter_title}') format = "MANGA" - + volume = re.findall(r"(?i)(?:Vol|v|volume)(?:\s|\.)?(?:\s|\.)?([0-9]+(?:\.[0-9]+)?)",chapter_title) + if volume: + volume = volume[0] + else: + volume = None # If "chapter" is in the chapter substring try: if not hasNumbers(chapter_title): @@ -159,11 +163,7 @@ def filename_parser(filename, logging_info): LOG.debug(f'chapter_number: {chapter_number}') logging_info['chapter_number'] = chapter_number - volume = re.findall(r"(?:Vol|v|volume)(?:\s|\.)?(?:\s|\.)?([0-9]+(?:\.[0-9]+)?)",chapter_title) - if volume: - volume = volume[0] - else: - volume = None + return manga_title, chapter_number, format, volume From 6559d2b2f55d73c3a4f0b32b8470b22813d0ab0f Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 16 May 2022 00:54:24 +0200 Subject: [PATCH 04/15] Localized_series tag was created but not populated --- MangaTaggerLib/MangaTaggerLib.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/MangaTaggerLib/MangaTaggerLib.py b/MangaTaggerLib/MangaTaggerLib.py index ec6c69a..ced68fc 100644 --- a/MangaTaggerLib/MangaTaggerLib.py +++ b/MangaTaggerLib/MangaTaggerLib.py @@ -493,11 +493,9 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info,volume_number series = SubElement(comicinfo, 'Series') series.text = metadata.series_title - if metadata.series_title_eng is not None and compare(metadata.series_title, metadata.series_title_eng) != 1: - # alt_series = SubElement(comicinfo, 'AlternateSeries') - # alt_series.text = metadata.series_title_eng - # if metadata.series_title_eng is not None: + if metadata.series_title_eng is not None and compare(metadata.series_title, metadata.series_title_eng) != 1 and metadata.series_title_eng!="": localized_series = SubElement(comicinfo,'LocalizedSeries') + localized_series.text = metadata.series_title_eng number = SubElement(comicinfo, 'Number') number.text = f'{chapter_number}' From 634efc660185261bda13346ba6716e068efb7cff Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 16 May 2022 10:06:49 +0200 Subject: [PATCH 05/15] MT will now try to parse Ch.## before anything else --- MangaTaggerLib/MangaTaggerLib.py | 12 ++++++++++-- tests/test_manga.py | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/MangaTaggerLib/MangaTaggerLib.py b/MangaTaggerLib/MangaTaggerLib.py index ced68fc..da80121 100644 --- a/MangaTaggerLib/MangaTaggerLib.py +++ b/MangaTaggerLib/MangaTaggerLib.py @@ -101,6 +101,12 @@ def filename_parser(filename, logging_info): volume = volume[0] else: volume = None + + chapter = re.findall(r"(?i)(?:(?:ch|chapter|c)(?:\s|\.)?(?:\s|\.)?(?:([0-9]+(?:\.[0-9]+)?)+(?:-([0-9]+(?:\.[0-9]+)?))?))",chapter_title) + if chapter: + chapter = f"{chapter[0][0]}" + else: + chapter = None # If "chapter" is in the chapter substring try: if not hasNumbers(chapter_title): @@ -163,8 +169,10 @@ def filename_parser(filename, logging_info): LOG.debug(f'chapter_number: {chapter_number}') logging_info['chapter_number'] = chapter_number - - return manga_title, chapter_number, format, volume + if chapter is not None: + return manga_title, chapter, format, volume + else: + return manga_title, chapter_number, format, volume def rename_action(current_file_path: Path, new_file_path: Path, manga_title, chapter_number, logging_info): diff --git a/tests/test_manga.py b/tests/test_manga.py index e04cf9b..b57c0ca 100644 --- a/tests/test_manga.py +++ b/tests/test_manga.py @@ -54,6 +54,14 @@ def test_filename_parser_ignore_fluff(self): result = filename_parser(filename, logging_info) self.assertEqual(expected_result, result) + def test_filename_parser_ignore_fluff_2(self): + ## Ignore Volume, chapter name and (part) + filename = "Kuma Kuma Kuma Bear -.- Ch. 064 - Kuma-san and the Shop's Opening Day 2.cbz" + directory_name = "Kuma Kuma Kuma Bear" + logging_info = {'event_id': 0, 'manga_title': directory_name, "original_filename": filename} + expected_result = ("Kuma Kuma Kuma Bear", "064", "MANGA", None) + result = filename_parser(filename, logging_info) + self.assertEqual(expected_result, result) class TestMangaRenameAction(unittest.TestCase): download_dir = Path('tests/downloads') library_dir = Path('tests/library') From ff18a29f0ed84aa38fc9d431cf17ea11958c192d Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Sat, 21 May 2022 05:13:23 +0200 Subject: [PATCH 06/15] Updated docker-image.yml --- .github/workflows/docker-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index baa7b89..14e36e8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,9 +1,9 @@ -name: ci - +name: Build & Publish on: push: branches: [ main, develop ] - +env: + IMAGE_NAME: "thepromidius/manga-tagger" jobs: docker_dev: name: Build Nightly Docker if Develop push @@ -23,7 +23,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - tags: promidius/manga-tagger:nightly + tags: ${{ env.IMAGE_NAME }}:nightly docker_stable: name: Build Stable Docker if Main push runs-on: ubuntu-latest @@ -46,4 +46,4 @@ jobs: uses: docker/build-push-action@v3 with: push: true - tags: promidius/manga-tagger:latest + tags: ${{ env.IMAGE_NAME }}:latest \ No newline at end of file From 49f879427c2be0f2ded77f1ecf7055ba3ef1176a Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 09:28:14 +0200 Subject: [PATCH 07/15] Updated docker-image.yml and README.md. Change cover disabled by default --- Dockerfile | 4 ++-- README.md | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78de574..01f8f08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ FROM ghcr.io/linuxserver/baseimage-alpine:3.13 LABEL \ - maintainer="TKVictor-Hang@outlook.fr" + maintainer="thepromidiusyt@gmail.com" ### Set default Timezone, overwrite default MangaTagger settings for the container ### ENV \ TZ="Europe/Paris" \ MANGA_TAGGER_DEBUG_MODE=false \ MANGA_TAGGER_DATA_DIR="/config/data" \ - MANGA_TAGGER_IMAGE_COVER=true \ + MANGA_TAGGER_IMAGE_COVER=false \ MANGA_TAGGER_IMAGE_DIR="/config/cover" \ MANGA_TAGGER_ADULT_RESULT=false \ MANGA_TAGGER_DOWNLOAD_DIR="/downloads" \ diff --git a/README.md b/README.md index a4360a6..91897ff 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ [![mt-hub-img]][mt-hub-lnk] -## Banh-Canh/Manga-Tagger +## ThePromidius/Manga-Tagger ## Descriptions This fork doesn't require FMD2. Running MangaTagger.py will make it watch the directory configured in the settings.json. Intended to be used in a docker container: -https://hub.docker.com/r/banhcanh/manga-tagger +https://hub.docker.com/r/thepromidius/manga-tagger input Files still have to be named like this (they can be in their own %MANGA% directory, or not) : %MANGA% -.- %CHAPTER%.cbz @@ -21,6 +21,9 @@ input Files still have to be named like this (they can be in their own %MANGA% d * Manga specific configuration More infos: +Check the WIKI: +https://github.com/ThePromidius/Manga-Tagger/wiki + https://github.com/Inpacchi/Manga-Tagger ## Downloading and Running Manga-Tagger @@ -49,7 +52,7 @@ python MangaTagger.py version: "2.1" services: mangatagger: - image: banhcanh/manga-tagger + image: thepromidius/manga-tagger container_name: mangatagger environment: - PUID=1000 @@ -109,9 +112,9 @@ services: restart: unless-stopped ``` -I recommend using this with my FMD2 docker image: https://hub.docker.com/r/banhcanh/docker-fmd2 +I recommend using this banhcanh docker image: https://hub.docker.com/r/banhcanh/docker-fmd2 -Environnement Variables overwrite the settings.json. In docker, it is only possible to configure with environnement variables. +Environment Variables overwrite the settings.json. In docker, it is only possible to configure with environment variables. Enabling adult result may give wrong manga match. Make sure the input manga title is as accurate as possible if enabling this or it may confuse Anilist's search. @@ -135,5 +138,5 @@ In this case, this title isn't accurate enough to search on Anilist and this why [MIT](https://choosealicense.com/licenses/mit/) -[mt-hub-img]: https://img.shields.io/docker/pulls/banhcanh/manga-tagger.svg -[mt-hub-lnk]: https://hub.docker.com/r/banhcanh/manga-tagger +[mt-hub-img]: https://img.shields.io/docker/pulls/thepromidius/manga-tagger.svg +[mt-hub-lnk]: https://hub.docker.com/r/thepromidius/manga-tagger From 9051184b4900749ad3e44419f77b4a870ebbbb20 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 10:49:02 +0200 Subject: [PATCH 08/15] Added volumes field. When finished number of volumes. Else None TODO: Save to database and save to comicinfo --- MangaTaggerLib/MangaTaggerLib.py | 110 ++++++++++++++++++------------- MangaTaggerLib/api.py | 1 + MangaTaggerLib/models.py | 7 ++ 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/MangaTaggerLib/MangaTaggerLib.py b/MangaTaggerLib/MangaTaggerLib.py index da80121..079a233 100644 --- a/MangaTaggerLib/MangaTaggerLib.py +++ b/MangaTaggerLib/MangaTaggerLib.py @@ -65,7 +65,7 @@ def process_manga_chapter(file_path: Path, event_id): manga_details = filename_parser(filename, logging_info) - metadata_tagger(file_path, manga_details[0], manga_details[1], manga_details[2], logging_info,manga_details[3]) + metadata_tagger(file_path, manga_details[0], manga_details[1], manga_details[2], logging_info, manga_details[3]) # Remove manga directory if empty try: @@ -76,6 +76,7 @@ def process_manga_chapter(file_path: Path, event_id): except OSError as e: LOG.info("Error: %s : %s" % (directory_path, e.strerror)) + def filename_parser(filename, logging_info): LOG.info(f'Attempting to rename "{filename}"...', extra=logging_info) @@ -96,13 +97,15 @@ def filename_parser(filename, logging_info): LOG.debug(f'chapter: {chapter_title}') format = "MANGA" - volume = re.findall(r"(?i)(?:Vol|v|volume)(?:\s|\.)?(?:\s|\.)?([0-9]+(?:\.[0-9]+)?)",chapter_title) + volume = re.findall(r"(?i)(?:Vol|v|volume)(?:\s|\.)?(?:\s|\.)?([0-9]+(?:\.[0-9]+)?)", chapter_title) if volume: volume = volume[0] else: volume = None - chapter = re.findall(r"(?i)(?:(?:ch|chapter|c)(?:\s|\.)?(?:\s|\.)?(?:([0-9]+(?:\.[0-9]+)?)+(?:-([0-9]+(?:\.[0-9]+)?))?))",chapter_title) + chapter = re.findall( + r"(?i)(?:(?:ch|chapter|c)(?:\s|\.)?(?:\s|\.)?(?:([0-9]+(?:\.[0-9]+)?)+(?:-([0-9]+(?:\.[0-9]+)?))?))", + chapter_title) if chapter: chapter = f"{chapter[0][0]}" else: @@ -110,9 +113,9 @@ def filename_parser(filename, logging_info): # If "chapter" is in the chapter substring try: if not hasNumbers(chapter_title): - if "oneshot" in chapter_title.lower(): - format = "ONE_SHOT" - chapter_title = "chap000" + if "oneshot" in chapter_title.lower(): + format = "ONE_SHOT" + chapter_title = "chap000" if "prologue" in chapter_title.lower(): chapter_title = chapter_title.replace(' ', '') @@ -125,15 +128,15 @@ def filename_parser(filename, logging_info): chapter_title = re.sub('\D*$', '', chapter_title) # Removed space and any character at the end of the chapter_title that are not number. Usually that's the name of the chapter. - # Match "Chapter5" "GAME005" "Page/005" "ACT-50" "#505" "V05.5CHAP5.5" without the chapter number, we removed spaces above + # Match "Chapter5" "GAME005" "Page/005" "ACT-50" "#505" "V05.5CHAP5.5" without the chapter number, we removed spaces above chapter_title_pattern = "[^\d\.]\D*\d*[.,]?\d*[^\d\.]\D*" if re.match(chapter_title_pattern, chapter_title): - p = re.compile(chapter_title_pattern) - prog = p.match(chapter_title) - chapter_title_name = prog.group(0) - delimiter = chapter_title_name - delimiter_index = len(chapter_title_name) + p = re.compile(chapter_title_pattern) + prog = p.match(chapter_title) + chapter_title_name = prog.group(0) + delimiter = chapter_title_name + delimiter_index = len(chapter_title_name) else: raise UnparsableFilenameError(filename, 'ch/chapter') except UnparsableFilenameError as ufe: @@ -187,7 +190,7 @@ def rename_action(current_file_path: Path, new_file_path: Path, manga_title, cha shutil.move(current_file_path, new_file_path) LOG.info(f'"{new_file_path.name.strip(".cbz")}" has been renamed.', extra=logging_info) ProcFilesTable.insert_record(current_file_path, new_file_path, manga_title, chapter_number, - logging_info) + logging_info) else: versions = ['v2', 'v3', 'v4', 'v5'] @@ -268,7 +271,7 @@ def compare_versions(old_filename: str, new_filename: str): return False -def metadata_tagger(file_path, manga_title, manga_chapter_number, format, logging_info,volume): +def metadata_tagger(file_path, manga_title, manga_chapter_number, format, logging_info, volume): manga_search = None db_exists = True retries = 0 @@ -285,7 +288,7 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin if exceptions[manga_title]['format'] == "MANGA" or exceptions[manga_title]['format'] == "ONE_SHOT": format = exceptions[manga_title]['format'] if exceptions[manga_title]['adult'] is True or exceptions[manga_title]['adult'] is False: - isadult= exceptions[manga_title]['adult'] + isadult = exceptions[manga_title]['adult'] manga_title = exceptions[manga_title]['anilist_title'] LOG.info(f'Table search value is "{manga_title}"', extra=logging_info) @@ -306,9 +309,10 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin LOG.info('Manga was not found in the database; resorting to Anilist API.', extra=logging_info) try: - if isadult: # enable adult result in Anilist + if isadult: # enable adult result in Anilist LOG.info('Adult result enabled') - manga_search = AniList.search_for_manga_title_by_manga_title_with_adult(manga_title, format, logging_info) + manga_search = AniList.search_for_manga_title_by_manga_title_with_adult(manga_title, format, + logging_info) else: manga_search = AniList.search_for_manga_title_by_manga_title(manga_title, format, logging_info) except (APIException, ConnectionError) as e: @@ -316,9 +320,10 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin LOG.warning('Manga Tagger has unintentionally breached the API limits on Anilist. Waiting 60s to clear ' 'all rate limiting limits...') time.sleep(60) - if isadult: # enable adult result in Anilist + if isadult: # enable adult result in Anilist LOG.info('Adult result enabled') - manga_search = AniList.search_for_manga_title_by_manga_title_with_adult(manga_title, format, logging_info) + manga_search = AniList.search_for_manga_title_by_manga_title_with_adult(manga_title, format, + logging_info) else: manga_search = AniList.search_for_manga_title_by_manga_title(manga_title, format, logging_info) if manga_search is None: @@ -342,23 +347,24 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin if AppSettings.mode_settings is None or AppSettings.mode_settings['rename_file']: if not manga_library_dir.exists(): - LOG.info(f'A directory for "{series_title}" in "{AppSettings.library_dir}" does not exist; creating now.') + LOG.info( + f'A directory for "{series_title}" in "{AppSettings.library_dir}" does not exist; creating now.') manga_library_dir.mkdir() try: - # Multithreading Optimization + # Multithreading Optimization if new_file_path in CURRENTLY_PENDING_RENAME: LOG.info(f'A file is currently being renamed under the filename "{new_filename}". Locking ' - f'{file_path} from further processing until this rename action is complete...', - extra=logging_info) + f'{file_path} from further processing until this rename action is complete...', + extra=logging_info) while new_file_path in CURRENTLY_PENDING_RENAME: time.sleep(1) LOG.info(f'The file being renamed to "{new_file_path}" has been completed. Unlocking ' - f'"{new_filename}" for file rename processing.', extra=logging_info) + f'"{new_filename}" for file rename processing.', extra=logging_info) else: LOG.info(f'No files currently currently being processed under the filename ' - f'"{new_filename}". Locking new filename for processing...', extra=logging_info) + f'"{new_filename}". Locking new filename for processing...', extra=logging_info) CURRENTLY_PENDING_RENAME.add(new_file_path) rename_action(file_path, new_file_path, series_title, manga_chapter_number, logging_info) @@ -375,7 +381,8 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin ProcSeriesTable.processed_series.add(manga_title) if AppSettings.image and not Path(f'{AppSettings.image_dir}/{series_title}_cover.jpg').exists(): - LOG.info(f'Image directory configured but cover not found. Send request to Anilist for necessary data.', extra=logging_info) + LOG.info(f'Image directory configured but cover not found. Send request to Anilist for necessary data.', + extra=logging_info) manga_id = MetadataTable.search_id_by_search_value(series_title) anilist_details = AniList.search_staff_by_mal_id(manga_id, logging_info) @@ -414,27 +421,28 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin LOG.debug(f'new_file_path: {new_file_path}') LOG.info(f'Checking for current and previously processed files with filename "{new_filename}"...', - extra=logging_info) + extra=logging_info) if AppSettings.mode_settings is None or AppSettings.mode_settings['rename_file']: if not manga_library_dir.exists(): - LOG.info(f'A directory for "{series_title}" in "{AppSettings.library_dir}" does not exist; creating now.') + LOG.info( + f'A directory for "{series_title}" in "{AppSettings.library_dir}" does not exist; creating now.') manga_library_dir.mkdir() try: - # Multithreading Optimization + # Multithreading Optimization if new_file_path in CURRENTLY_PENDING_RENAME: LOG.info(f'A file is currently being renamed under the filename "{new_filename}". Locking ' - f'{file_path} from further processing until this rename action is complete...', - extra=logging_info) + f'{file_path} from further processing until this rename action is complete...', + extra=logging_info) while new_file_path in CURRENTLY_PENDING_RENAME: time.sleep(1) LOG.info(f'The file being renamed to "{new_file_path}" has been completed. Unlocking ' - f'"{new_filename}" for file rename processing.', extra=logging_info) + f'"{new_filename}" for file rename processing.', extra=logging_info) else: LOG.info(f'No files currently currently being processed under the filename ' - f'"{new_filename}". Locking new filename for processing...', extra=logging_info) + f'"{new_filename}". Locking new filename for processing...', extra=logging_info) CURRENTLY_PENDING_RENAME.add(new_file_path) rename_action(file_path, new_file_path, series_title, manga_chapter_number, logging_info) @@ -448,13 +456,15 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin logging_info['metadata'] = manga_metadata.__dict__ if series_title in ProcSeriesTable.processed_series: - LOG.info(f'Found an entry in manga_metadata for "{series_title}". Filename was probably not perfectly named according to MAL. Not adding metadata to MetadataTable.', extra=logging_info) + LOG.info( + f'Found an entry in manga_metadata for "{series_title}". Filename was probably not perfectly named according to MAL. Not adding metadata to MetadataTable.', + extra=logging_info) else: if AppSettings.mode_settings is None or ('database_insert' in AppSettings.mode_settings.keys() - and AppSettings.mode_settings['database_insert']): - MetadataTable.insert(manga_metadata, logging_info) + and AppSettings.mode_settings['database_insert']): + MetadataTable.insert(manga_metadata, logging_info) LOG.info(f'Retrieved metadata for "{series_title}" from the Anilist and MyAnimeList APIs; ' - f'now unlocking series for processing!', extra=logging_info) + f'now unlocking series for processing!', extra=logging_info) ProcSeriesTable.processed_series.add(series_title) if AppSettings.mode_settings is None or ('write_comicinfo' in AppSettings.mode_settings.keys() @@ -469,12 +479,13 @@ def metadata_tagger(file_path, manga_title, manga_chapter_number, format, loggin else: LOG.info('Image Directory not set, not downloading series cover image.', extra=logging_info) - comicinfo_xml = construct_comicinfo_xml(manga_metadata, manga_chapter_number, logging_info,volume) + comicinfo_xml = construct_comicinfo_xml(manga_metadata, manga_chapter_number, logging_info, volume) reconstruct_manga_chapter(series_title, comicinfo_xml, new_file_path, logging_info) LOG.info(f'Processing on "{new_file_path}" has finished.', extra=logging_info) return manga_metadata + def construct_anilist_titles(anilist_details): anilist_titles = {} @@ -489,7 +500,8 @@ def construct_anilist_titles(anilist_details): return anilist_titles -def construct_comicinfo_xml(metadata, chapter_number, logging_info,volume_number): + +def construct_comicinfo_xml(metadata, chapter_number, logging_info, volume_number): LOG.info(f'Constructing comicinfo object for "{metadata.series_title}", chapter {chapter_number}...', extra=logging_info) @@ -501,8 +513,9 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info,volume_number series = SubElement(comicinfo, 'Series') series.text = metadata.series_title - if metadata.series_title_eng is not None and compare(metadata.series_title, metadata.series_title_eng) != 1 and metadata.series_title_eng!="": - localized_series = SubElement(comicinfo,'LocalizedSeries') + if metadata.series_title_eng is not None and compare(metadata.series_title, + metadata.series_title_eng) != 1 and metadata.series_title_eng != "": + localized_series = SubElement(comicinfo, 'LocalizedSeries') localized_series.text = metadata.series_title_eng number = SubElement(comicinfo, 'Number') @@ -540,8 +553,8 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info,volume_number cover_artist = SubElement(comicinfo, 'CoverArtist') cover_artist.text = next(iter(metadata.staff['art'])) -# publisher = SubElement(comicinfo, 'Publisher') -# publisher.text = next(iter(metadata.serializations)) + # publisher = SubElement(comicinfo, 'Publisher') + # publisher.text = next(iter(metadata.serializations)) genre = SubElement(comicinfo, 'Genre') for mg in metadata.genres: @@ -550,8 +563,8 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info,volume_number else: genre.text = f'{mg}' -# web = SubElement(comicinfo, 'Web') -# web.text = metadata.mal_url + # web = SubElement(comicinfo, 'Web') + # web.text = metadata.mal_url language = SubElement(comicinfo, 'LanguageISO') language.text = 'en' @@ -584,11 +597,13 @@ def reconstruct_manga_chapter(manga_title, comicinfo_xml, manga_file_path, loggi LOG.info(f'ComicInfo.xml has been created and appended to "{manga_file_path}".', extra=logging_info) + def download_cover_image(manga_title, image_url): image = requests.get(image_url) with open(f'{AppSettings.image_dir}/{manga_title}_cover.jpg', 'wb') as image_file: image_file.write(image.content) - + + def slugify(value, allow_unicode=False): """ Taken from https://github.com/django/django/blob/master/django/utils/text.py @@ -604,6 +619,7 @@ def slugify(value, allow_unicode=False): value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') value = re.sub(r'[^\w\s-]', '', value) return value - + + def hasNumbers(inputString): return bool(re.search(r'\d', inputString)) diff --git a/MangaTaggerLib/api.py b/MangaTaggerLib/api.py index 3dd0c20..cf37743 100644 --- a/MangaTaggerLib/api.py +++ b/MangaTaggerLib/api.py @@ -78,6 +78,7 @@ def search_details_by_series_id(cls, series_id, format, logging_info): Media (id: $series_id, type: MANGA, format: $format) { id status + volumes siteUrl title { romaji diff --git a/MangaTaggerLib/models.py b/MangaTaggerLib/models.py index df5bd2f..d5db310 100644 --- a/MangaTaggerLib/models.py +++ b/MangaTaggerLib/models.py @@ -45,6 +45,11 @@ def _construct_api_metadata(self, anilist_details, logging_info): self.series_title_jap = anilist_details['title']['native'] self.status = anilist_details['status'] + if anilist_details.get('volumes'): + self.volumes = anilist_details.get('volumes') + else: + self.volumes = None + self.type = anilist_details['type'] self.description = anilist_details['description'] self.anilist_url = anilist_details['siteUrl'] @@ -64,6 +69,7 @@ def _construct_database_metadata(self, details): self.series_title_eng = details['series_title_eng'] self.series_title_jap = details['series_title_jap'] self.status = details['status'] + self.volumes = details.get("volumes") self.type = details['type'] self.description = details['description'] self.anilist_url = details['anilist_url'] @@ -160,6 +166,7 @@ def test_value(self): 'series_title_eng': self.series_title_eng, 'series_title_jap': self.series_title_jap, 'status': self.status, + 'volumes':self.volumes, # 'mal_url': self.mal_url, 'anilist_url': self.anilist_url, 'publish_date': self.publish_date, From d71e8a4aa2fa69a6287ca1b31aad07c0abc0f82e Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 11:15:38 +0200 Subject: [PATCH 09/15] Updated unit tests data --- tests/data/3D Kanojo Real Girl/data.json | 1 + tests/data/BLEACH/data.json | 319 +++++++++++++++++++++-- tests/data/Hurejasik/data.json | 87 ++++--- tests/data/Naruto/data.json | 3 +- tests/test_integration.py | 14 +- 5 files changed, 345 insertions(+), 79 deletions(-) diff --git a/tests/data/3D Kanojo Real Girl/data.json b/tests/data/3D Kanojo Real Girl/data.json index 26ce567..baf79a9 100644 --- a/tests/data/3D Kanojo Real Girl/data.json +++ b/tests/data/3D Kanojo Real Girl/data.json @@ -1,6 +1,7 @@ { "id": 80767, "status": "FINISHED", + "volumes": 12, "siteUrl": "https://anilist.co/manga/80767", "title": { "romaji": "3D Kanojo: Real Girl", diff --git a/tests/data/BLEACH/data.json b/tests/data/BLEACH/data.json index ea3475d..2d74297 100644 --- a/tests/data/BLEACH/data.json +++ b/tests/data/BLEACH/data.json @@ -1,44 +1,307 @@ { "id": 30012, "status": "FINISHED", + "volumes": 74, "siteUrl": "https://anilist.co/manga/30012", "title": { - "romaji": "BLEACH", - "english": "Bleach", - "native": "ブリーチ" + "romaji": "BLEACH", + "english": "Bleach", + "native": "BLEACH" }, "type": "MANGA", "genres": [ - "Action", - "Adventure", - "Supernatural" + "Action", + "Adventure", + "Supernatural" ], "startDate": { - "day": 7, - "month": 8, - "year": 2001 + "day": 7, + "month": 8, + "year": 2001 }, "coverImage": { - "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/bx30012-z7U138mUaPdN.png" + "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/bx30012-z7U138mUaPdN.png" }, "staff": { - "edges": [ - { - "node": { - "name": { - "first": "Tite", - "last": "Kubo", - "full": "Tite Kubo", - "alternative": [ - "Taito Kubo", - "Noriaki Kubo (久保宣章)" - ] - }, - "siteUrl": "https://anilist.co/staff/96880" - }, - "role": "Story & Art" - } - ] + "edges": [ + { + "node": { + "name": { + "first": "Tite", + "last": "Kubo", + "full": "Tite Kubo", + "alternative": [ + "Taito Kubo", + "Noriaki Kubo (\u4e45\u4fdd\u5ba3\u7ae0)" + ] + }, + "siteUrl": "https://anilist.co/staff/96880" + }, + "role": "Story & Art" + }, + { + "node": { + "name": { + "first": "Ricardo", + "last": "Cruz", + "full": "Ricardo Cruz", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/104809" + }, + "role": "Translator (Portuguese: vols 1-24)" + }, + { + "node": { + "name": { + "first": "Drik", + "last": "Sada", + "full": "Drik Sada", + "alternative": [ + "Adriana Kazue Sada" + ] + }, + "siteUrl": "https://anilist.co/staff/220675" + }, + "role": "Translator (Portuguese: vols 25-52, 71-74)" + }, + { + "node": { + "name": { + "first": "Christine", + "last": "Dashiell", + "full": "Christine Dashiell", + "alternative": [ + "Christine Schilling" + ] + }, + "siteUrl": "https://anilist.co/staff/219441" + }, + "role": "Translator (English: vol 49)" + }, + { + "node": { + "name": { + "first": "Joe", + "last": "Yamazaki", + "full": "Joe Yamazaki", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/218469" + }, + "role": "Translator (English)" + }, + { + "node": { + "name": { + "first": "Mark", + "last": "McMurray", + "full": "Mark McMurray", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/224874" + }, + "role": "Touch-up Art & Lettering (English: vols 16, 20)" + }, + { + "node": { + "name": { + "first": "Mark", + "last": "McMurray", + "full": "Mark McMurray", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/224874" + }, + "role": "Lettering (English: vols: 16, 17, 19, 20, 22, 23, 24)" + }, + { + "node": { + "name": { + "first": "Evan", + "last": "Waldinger", + "full": "Evan Waldinger", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/227608" + }, + "role": "Touch-up Art & Lettering (English: vol 21)" + }, + { + "node": { + "name": { + "first": "Dave", + "last": "Lanphear", + "full": "Dave Lanphear", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/227606" + }, + "role": "Touch-up Art & Lettering (English: vols 3, 5, 6, 7, 8)" + }, + { + "node": { + "name": { + "first": "Andy", + "last": "Ristaino", + "full": "Andy Ristaino", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/227607" + }, + "role": "Touch-up Art & Lettering (English)" + }, + { + "node": { + "name": { + "first": "Marta", + "last": "Gallego", + "full": "Marta Gallego", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/225332" + }, + "role": "Translator (Spanish)" + }, + { + "node": { + "name": { + "first": "Marc", + "last": "Bernab\u00e9", + "full": "Marc Bernab\u00e9", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/214718" + }, + "role": "Translator (Spanish)" + }, + { + "node": { + "name": { + "first": "Ver\u00f3nica", + "last": "Calafell", + "full": "Ver\u00f3nica Calafell", + "alternative": [ + "Ver\u00f3nica Calafell Callejo" + ] + }, + "siteUrl": "https://anilist.co/staff/225627" + }, + "role": "Translator (Spanish)" + }, + { + "node": { + "name": { + "first": "Simona", + "last": "Stanzani", + "full": "Simona Stanzani", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/225407" + }, + "role": "Translator (Italian)" + }, + { + "node": { + "name": { + "first": "Rafael", + "last": "Morata", + "full": "Rafael Morata", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/227609" + }, + "role": "Translator (Spanish)" + }, + { + "node": { + "name": { + "first": "Daniel", + "last": "B\u00fcchner", + "full": "Daniel B\u00fcchner", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/227610" + }, + "role": "Translator (German)" + }, + { + "node": { + "name": { + "first": "Kentarou", + "last": "Kurimoto", + "full": "Kentarou Kurimoto", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/119236" + }, + "role": "Assistant" + }, + { + "node": { + "name": { + "first": "Shou", + "last": "Aimoto", + "full": "Shou Aimoto", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/98753" + }, + "role": "Assistant" + }, + { + "node": { + "name": { + "first": "Hitoshi", + "last": "Imoto", + "full": "Hitoshi Imoto", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/126703" + }, + "role": "Assistant" + }, + { + "node": { + "name": { + "first": "Anne-Sophie", + "last": "Thevenon", + "full": "Anne-Sophie Thevenon", + "alternative": [ + "Anne-Sophie Th\u00e9venon" + ] + }, + "siteUrl": "https://anilist.co/staff/236835" + }, + "role": "Translator (French)" + }, + { + "node": { + "name": { + "first": "Shou", + "last": "Shiromoto", + "full": "Shou Shiromoto", + "alternative": [] + }, + "siteUrl": "https://anilist.co/staff/249345" + }, + "role": "Assistant" + }, + { + "node": { + "name": { + "first": "Tayfun", + "last": "Nitahara Haks\u00f6yliyen", + "full": "Tayfun Nitahara Haks\u00f6yliyen", + "alternative": [ + "Tayfun Nitahara Haksoyliyen" + ] + }, + "siteUrl": "https://anilist.co/staff/275233" + }, + "role": "Translator (Turkish: vols 1-17)" + } + ] }, - "description": "Ichigo Kurosaki has always been able to see ghosts, but this ability doesn't change his life nearly as much as his close encounter with Rukia Kuchiki, a Soul Reaper and member of the mysterious Soul Society. While fighting a Hollow, an evil spirit that preys on humans who display psychic energy, Rukia attempts to lend Ichigo some of her powers so that he can save his family; but much to her surprise, Ichigo absorbs every last drop of her energy. Now a full-fledged Soul Reaper himself, Ichigo quickly learns that the world he inhabits is one full of dangerous spirits and, along with Rukia— who is slowly regaining her powers— it's Ichigo's job to protect the innocent from Hollows and help the spirits themselves find peace.

\n(Source: Anime News Network)

\n20 Included bonus chapters:
\nVolume 10 - Chapter 88.5: Karakura Super Heroes.
\nVolume 12 - Chapter 0.8: A Wonderful Error.
\nVolume 15 - Chapter -17: Prelude for the Straying Stars.
\nVolume 20 - Chapter -12.5: Blooming Under a Cold Moon.
\nVolume 23 - Chapter 0.Side-A: The Sand; Chapter 0.Side-B: The Rotator.
\nVolume 32 - Chapter -16: Death on the Ice Field.
\nVolume 36 - Chapters -108 to -100: Turn Back the Pendulum (1-9).
\nVolume 37 - Chapters -99 to -98: Turn Back the Pendulum (10-11); Chapter -97: Let Stop the Pendulum.
\nVolume 70 - Chapter 520.5: Walk Under Two Letters." + "description": "Ichigo Kurosaki has always been able to see ghosts, but this ability doesn't change his life nearly as much as his close encounter with Rukia Kuchiki, a Soul Reaper and member of the mysterious Soul Society. While fighting a Hollow, an evil spirit that preys on humans who display psychic energy, Rukia attempts to lend Ichigo some of her powers so that he can save his family; but much to her surprise, Ichigo absorbs every last drop of her energy. Now a full-fledged Soul Reaper himself, Ichigo quickly learns that the world he inhabits is one full of dangerous spirits and, along with Rukia\u2014 who is slowly regaining her powers\u2014 it's Ichigo's job to protect the innocent from Hollows and help the spirits themselves find peace.

\n(Source: Anime News Network)

\nNote: Chapter count includes the 12-chapter \u201cTurn Back The Pendulum\u201d side story and 8 extra chapters." } diff --git a/tests/data/Hurejasik/data.json b/tests/data/Hurejasik/data.json index 2b3a02c..068dd5d 100644 --- a/tests/data/Hurejasik/data.json +++ b/tests/data/Hurejasik/data.json @@ -1,60 +1,61 @@ { "id": 86964, "status": "FINISHED", + "volumes": 5, "siteUrl": "https://anilist.co/manga/86964", "title": { - "romaji": "Hurejasik", - "english": "Bastard", - "native": "후레자식" + "romaji": "Hurejasik", + "english": "Bastard", + "native": "후레자식" }, "type": "MANGA", "genres": [ - "Drama", - "Horror", - "Mystery", - "Psychological", - "Romance", - "Thriller" + "Drama", + "Horror", + "Mystery", + "Psychological", + "Romance", + "Thriller" ], "startDate": { - "day": 4, - "month": 7, - "year": 2014 + "day": 4, + "month": 7, + "year": 2014 }, "coverImage": { - "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/nx86964-r7S3IbJNr4SD.jpg" + "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/nx86964-r7S3IbJNr4SD.jpg" }, "staff": { - "edges": [ - { - "node": { - "name": { - "first": "Youngchan", - "last": "Hwang", - "full": "Youngchan Hwang", - "alternative": [ - "" - ] - }, - "siteUrl": "https://anilist.co/staff/119717" - }, - "role": "Art" - }, - { - "node": { - "name": { - "first": "Carnby", - "last": "Kim", - "full": "Carnby Kim", - "alternative": [ - "김칸비" - ] - }, - "siteUrl": "https://anilist.co/staff/119718" - }, - "role": "Story" - } - ] + "edges": [ + { + "node": { + "name": { + "first": "Yeong-chan", + "last": "Hwang", + "full": "Yeong-chan Hwang", + "alternative": [ + "" + ] + }, + "siteUrl": "https://anilist.co/staff/119717" + }, + "role": "Art" + }, + { + "node": { + "name": { + "first": "Carnby", + "last": "Kim", + "full": "Carnby Kim", + "alternative": [ + "김칸비" + ] + }, + "siteUrl": "https://anilist.co/staff/119718" + }, + "role": "Story" + } + ] }, "description": "There is nowhere that Seon Jin can find solace. At school, he is ruthlessly bullied due to his unsettlingly quiet nature and weak appearance. However, this is not the source of Jin's insurmountable terror: the thing that he fears more than anything else is his own father.

\nTo most, Jin's father is a successful businessman, good samaritan, and doting parent. But that is merely a facade; in truth, he is a deranged serial killer—and Jin is his unwilling accomplice. For years, they have been carrying out this ruse with the police being none the wiser. However, when his father takes an interest in the pretty transfer student Yoon Kyun, Jin must make a decision—be the coward who sends her to the gallows like all the rest, or be the bastard of a son who defies his wicked parent.

\n(Source: MAL Rewrite)

\nNote: Includes the prologue." } diff --git a/tests/data/Naruto/data.json b/tests/data/Naruto/data.json index f14d17b..8f11139 100644 --- a/tests/data/Naruto/data.json +++ b/tests/data/Naruto/data.json @@ -1,9 +1,10 @@ { "id": 30011, "status": "FINISHED", + "volumes": 72, "siteUrl": "https://anilist.co/manga/30011", "title": { - "romaji": "Naruto", + "romaji": "NARUTO", "english": "Naruto", "native": "NARUTO -ナルト-" }, diff --git a/tests/test_integration.py b/tests/test_integration.py index 31a2b42..01bf1a6 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -47,7 +47,7 @@ def test_comicinfo_xml_creation_case_1(self): manga_metadata = Metadata(title, {}, anilist_details) - self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {})) + self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {},None)) def test_comicinfo_xml_creation_case_2(self): title = 'Naruto' @@ -59,7 +59,7 @@ def test_comicinfo_xml_creation_case_2(self): manga_metadata = Metadata(title, {}, anilist_details) - self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {})) + self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {},None)) def test_metadata_case_1(self): title = 'BLEACH' @@ -71,7 +71,7 @@ def test_metadata_case_1(self): anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {}) + actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {},None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) @@ -85,7 +85,7 @@ def test_metadata_case_2(self): anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {}) + actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {},None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) @@ -101,7 +101,7 @@ def test_metadata_case_3(self): anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {}) + actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {},None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) @@ -116,7 +116,7 @@ def test_metadata_case_4(self): anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {}) + actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {},None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) @@ -130,6 +130,6 @@ def test_metadata_case_5(self): anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "ONE_SHOT", {}) + actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "ONE_SHOT", {},None) self.assertNotEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) From 1e5ce824fd91c3b16279f56e696d3e9c884b0709 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 11:31:15 +0200 Subject: [PATCH 10/15] Added volume number to comicinfo. Added anilist_url to comicinfo web field. --- MangaTaggerLib/MangaTaggerLib.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/MangaTaggerLib/MangaTaggerLib.py b/MangaTaggerLib/MangaTaggerLib.py index 079a233..fc64bc6 100644 --- a/MangaTaggerLib/MangaTaggerLib.py +++ b/MangaTaggerLib/MangaTaggerLib.py @@ -501,7 +501,7 @@ def construct_anilist_titles(anilist_details): return anilist_titles -def construct_comicinfo_xml(metadata, chapter_number, logging_info, volume_number): +def construct_comicinfo_xml(metadata: Metadata, chapter_number, logging_info, volume_number): LOG.info(f'Constructing comicinfo object for "{metadata.series_title}", chapter {chapter_number}...', extra=logging_info) @@ -524,6 +524,10 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info, volume_numbe volume = SubElement(comicinfo, 'Volume') volume.text = f'{volume_number}' + if metadata.volumes is not None: + count = SubElement(comicinfo,"Count") + count.text = f'{metadata.volumes}' + summary = SubElement(comicinfo, 'Summary') soup = BeautifulSoup(metadata.description, "html.parser") summary.text = soup.get_text() @@ -563,8 +567,8 @@ def construct_comicinfo_xml(metadata, chapter_number, logging_info, volume_numbe else: genre.text = f'{mg}' - # web = SubElement(comicinfo, 'Web') - # web.text = metadata.mal_url + web = SubElement(comicinfo, 'Web') + web.text = metadata.anilist_url language = SubElement(comicinfo, 'LanguageISO') language.text = 'en' From f3f3ecc0aca3ebc59d87ed023351df50cd382f1d Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 11:58:50 +0200 Subject: [PATCH 11/15] Added tests CI --- .github/workflows/Run_Tests.yml | 28 +++++++++++++++++++++++++ tests/test_integration.py | 36 +++++++++++++++++---------------- 2 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/Run_Tests.yml diff --git a/.github/workflows/Run_Tests.yml b/.github/workflows/Run_Tests.yml new file mode 100644 index 0000000..1d760a9 --- /dev/null +++ b/.github/workflows/Run_Tests.yml @@ -0,0 +1,28 @@ +name: Python tests + +on: + - push + - pull_request + +jobs: + build: +# env: +# DISPLAY: ":99.0" + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ '3.7','3.8','3.9', '3.10' ] + name: Python ${{ matrix.python-version }} sample + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - run: pip install -r requirements.txt +# - run: sudo apt install xvfb +# - name: Start xvfb +# run: | +# Xvfb :99 -screen 0 1920x1080x24 &disown + - name: Run the tests + run: python3 -m unittest discover -s tests -t . diff --git a/tests/test_integration.py b/tests/test_integration.py index 01bf1a6..eda4072 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,5 +1,6 @@ import json import logging +import os import unittest from pathlib import Path from unittest.mock import patch @@ -25,6 +26,7 @@ def setUp(self) -> None: self.models_AppSettings = patch1.start() self.addCleanup(patch1.stop) self.models_AppSettings.timezone = 'America/New_York' + self.models_AppSettings.library_dir = os.getcwd() patch2 = patch('MangaTaggerLib.MangaTaggerLib.MetadataTable') self.MetadataTable = patch2.start() @@ -47,7 +49,7 @@ def test_comicinfo_xml_creation_case_1(self): manga_metadata = Metadata(title, {}, anilist_details) - self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {},None)) + self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {}, None)) def test_comicinfo_xml_creation_case_2(self): title = 'Naruto' @@ -59,33 +61,33 @@ def test_comicinfo_xml_creation_case_2(self): manga_metadata = Metadata(title, {}, anilist_details) - self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {},None)) + self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {}, None)) def test_metadata_case_1(self): title = 'BLEACH' - self.MangaTaggerLib_AppSettings.mode_settings = { 'write_comicinfo': False } - self.MangaTaggerLib_AppSettings.mode_settings = { 'rename_file': False } + self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} + self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {},None) + actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {}, None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) def test_metadata_case_2(self): title = 'Naruto' - self.MangaTaggerLib_AppSettings.mode_settings = { 'write_comicinfo': False } - self.MangaTaggerLib_AppSettings.mode_settings = { 'rename_file': False } + self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} + self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {},None) + actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {}, None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) @@ -93,15 +95,15 @@ def test_metadata_case_3(self): title = '3D Kanojo Real Girl' downloaded_title = '3D Kanojo' - self.MangaTaggerLib_AppSettings.mode_settings = { 'write_comicinfo': False } - self.MangaTaggerLib_AppSettings.mode_settings = { 'rename_file': False } + self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} + self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} self.MangaTaggerLib_AppSettings.adult_result = False with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {},None) + actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {}, None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) @@ -109,27 +111,27 @@ def test_metadata_case_4(self): title = 'Hurejasik' downloaded_title = 'Bastard' - self.MangaTaggerLib_AppSettings.mode_settings = { 'write_comicinfo': False } - self.MangaTaggerLib_AppSettings.mode_settings = { 'rename_file': False } + self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} + self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {},None) + actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {}, None) self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) def test_metadata_case_5(self): title = 'Naruto' - self.MangaTaggerLib_AppSettings.mode_settings = { 'write_comicinfo': False } - self.MangaTaggerLib_AppSettings.mode_settings = { 'rename_file': False } + self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} + self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: anilist_details = json.load(data) expected_manga_metadata = Metadata(title, {}, anilist_details) - actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "ONE_SHOT", {},None) + actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "ONE_SHOT", {}, None) self.assertNotEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) From c0982de7208a5376795c84b9a914dacc6b164c61 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 12:51:18 +0200 Subject: [PATCH 12/15] Removed Python 3.7. Incompatible tests --- .github/workflows/Run_Tests.yml | 2 +- tests/test_integration.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/Run_Tests.yml b/.github/workflows/Run_Tests.yml index 1d760a9..c8b278e 100644 --- a/.github/workflows/Run_Tests.yml +++ b/.github/workflows/Run_Tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.7','3.8','3.9', '3.10' ] + python-version: [ '3.8','3.9', '3.10' ] name: Python ${{ matrix.python-version }} sample steps: - uses: actions/checkout@v3 diff --git a/tests/test_integration.py b/tests/test_integration.py index eda4072..1b0a37d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -26,7 +26,6 @@ def setUp(self) -> None: self.models_AppSettings = patch1.start() self.addCleanup(patch1.stop) self.models_AppSettings.timezone = 'America/New_York' - self.models_AppSettings.library_dir = os.getcwd() patch2 = patch('MangaTaggerLib.MangaTaggerLib.MetadataTable') self.MetadataTable = patch2.start() From 5268b13098c924add0056c41c6f5851a89732767 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 12:54:47 +0200 Subject: [PATCH 13/15] Removed Python 3.10 Numpy not supported yet --- .github/workflows/Run_Tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Run_Tests.yml b/.github/workflows/Run_Tests.yml index c8b278e..884314e 100644 --- a/.github/workflows/Run_Tests.yml +++ b/.github/workflows/Run_Tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.8','3.9', '3.10' ] + python-version: [ '3.8','3.9', ] name: Python ${{ matrix.python-version }} sample steps: - uses: actions/checkout@v3 From a176d5d39c2bc819ff15a0f767199361308f8cb1 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 15:01:37 +0200 Subject: [PATCH 14/15] Updated docker-image.yml dev branch --- .github/workflows/docker-image.yml | 26 +++++++++++++++++++++++--- README.md | 6 ++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 14e36e8..d3c4ae8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,11 +1,11 @@ name: Build & Publish on: push: - branches: [ main, develop ] + branches: [ main, develop , dev ] env: IMAGE_NAME: "thepromidius/manga-tagger" jobs: - docker_dev: + docker_develop: name: Build Nightly Docker if Develop push runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} @@ -46,4 +46,24 @@ jobs: uses: docker/build-push-action@v3 with: push: true - tags: ${{ env.IMAGE_NAME }}:latest \ No newline at end of file + tags: ${{ env.IMAGE_NAME }}:latest + + docker_test: + name: Build test Docker image if test push + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/test' }} + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v3 + with: + push: true + tags: ${{ env.IMAGE_NAME }}:test diff --git a/README.md b/README.md index 91897ff..f96e4ce 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![mt-hub-img]][mt-hub-lnk] +[![Python tests](https://github.com/ThePromidius/Manga-Tagger/actions/workflows/Run_Tests.yml/badge.svg)](https://github.com/ThePromidius/Manga-Tagger/actions/workflows/Run_Tests.yml) +[![mt-hub-img]][mt-hub-lnk] ## ThePromidius/Manga-Tagger ## Descriptions @@ -8,7 +9,7 @@ This fork doesn't require FMD2. Running MangaTagger.py will make it watch the di Intended to be used in a docker container: https://hub.docker.com/r/thepromidius/manga-tagger -input Files still have to be named like this (they can be in their own %MANGA% directory, or not) : %MANGA% -.- %CHAPTER%.cbz +input Files still have to be named like this (they can be in their own %MANGA% directory, or not) : `%MANGA% -.- %CHAPTER%.cbz` ## Features: @@ -140,3 +141,4 @@ In this case, this title isn't accurate enough to search on Anilist and this why [mt-hub-img]: https://img.shields.io/docker/pulls/thepromidius/manga-tagger.svg [mt-hub-lnk]: https://hub.docker.com/r/thepromidius/manga-tagger + From bc683a8e5cd0a58053f738a7fb5f83693efe09f7 Mon Sep 17 00:00:00 2001 From: ThePromidius Date: Mon, 30 May 2022 16:20:13 +0200 Subject: [PATCH 15/15] Build test image on any event on test branch. --- .github/workflows/docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d3c4ae8..4efce24 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,7 +1,7 @@ name: Build & Publish on: push: - branches: [ main, develop , dev ] + branches: [ main, develop , test ] env: IMAGE_NAME: "thepromidius/manga-tagger" jobs: @@ -51,7 +51,7 @@ jobs: docker_test: name: Build test Docker image if test push runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/test' }} + if: ${{github.ref == 'refs/heads/test' }} steps: - name: Set up QEMU uses: docker/setup-qemu-action@v2