From c13d1465033bb97fc03d9b454aabab5ec11e0955 Mon Sep 17 00:00:00 2001 From: Dawn India Date: Fri, 11 Oct 2024 13:10:31 +0530 Subject: [PATCH] Minor fixes Fix errors from the last PR. - Sync MLTB. - Fix same directory stuck issue. - Add ability to use different folder names using -sd arg along with the links inside the batch. - Other minor fixes. - Fix some bugs in jdowloader while using multi. - Now you do jd sync while downloading (not while collecting)(not recommended). - Remove youtube links restriction in jdownloader (i didn't try youtube download with jd yet). - Minor fix for jdwonloader sync while download and when adding creds from bot settings. - Some fixes and improvemnts for jdownloader. - Fix file uplaod name (important). - Remove offline files auto from jd packages. - Fix ddl mediafire. - Trim path for jdownloader incase the path length is more than 255. - In case you got file error invalid download path then you must rename the file(s). - I have done the folder rename auto for you but i don't want to fetch all files names, this will take a lot of time. --- bot/helper/common.py | 32 +- bot/helper/ext_utils/bot_utils.py | 1 + bot/helper/ext_utils/help_messages.py | 39 +- bot/helper/ext_utils/jdownloader_booter.py | 15 +- bot/helper/ext_utils/media_utils.py | 48 +- bot/helper/listeners/jdownloader_listener.py | 116 ++- bot/helper/listeners/task_listener.py | 109 ++- bot/helper/listeners/ytdlp_listener.py | 3 +- .../download_utils/direct_link_generator.py | 57 +- .../task_utils/download_utils/jd_download.py | 660 +++++++++--------- .../status_utils/jdownloader_status.py | 6 +- .../status_utils/media_convert_status.py | 10 +- .../status_utils/sample_video_status.py | 16 +- bot/helper/telegram_helper/message_utils.py | 5 +- bot/modules/bot_settings.py | 54 +- bot/modules/clone.py | 1 - bot/modules/mirror_leech.py | 104 ++- bot/modules/ytdlp.py | 136 ++-- 18 files changed, 749 insertions(+), 663 deletions(-) diff --git a/bot/helper/common.py b/bot/helper/common.py index 94a5fe039fe0..853baa47dca5 100644 --- a/bot/helper/common.py +++ b/bot/helper/common.py @@ -126,6 +126,7 @@ def __init__(self): self.thumbnail_layout = "" self.metadata = None self.m_attachment = None + self.folder_name = "" self.get_chat = None self.split_size = 0 self.max_split_size = 0 @@ -776,7 +777,7 @@ async def get_tag(self, text: list): self.tag = self.user.id @new_task - async def run_multi(self, input_list, folder_name, obj): + async def run_multi(self, input_list, obj): if ( config_dict["DISABLE_MULTI"] and self.multi > 1 @@ -816,6 +817,9 @@ async def run_multi(self, input_list, folder_name, obj): self.message, # type: ignore smsg ) + async with task_dict_lock: + for fd_name in self.same_dir: # type: ignore + self.same_dir[fd_name]["total"] -= self.multi # type: ignore return if len(self.bulk) != 0: msg = input_list[:1] @@ -850,8 +854,6 @@ async def run_multi(self, input_list, folder_name, obj): chat_id=self.message.chat.id, # type: ignore message_ids=nextmsg.id # type: ignore ) - if folder_name: - self.same_dir["tasks"].add(nextmsg.id) # type: ignore if self.message.from_user: # type: ignore nextmsg.from_user = self.user else: @@ -901,9 +903,14 @@ async def init_bulk(self, input_list, bulk_start, bulk_end, obj): del self.options[index + 1] self.options = " ".join(self.options) b_msg.append(f"{self.bulk[0]} -m {len(self.bulk)} {self.options}") + msg = " ".join(b_msg) + if len(self.bulk) > 2: + self.multi_tag = token_urlsafe(3) + multi_tags.add(self.multi_tag) + msg += f"\nCancel Multi: /{BotCommands.CancelTaskCommand[1]} {self.multi_tag}" nextmsg = await send_message( self.message, # type: ignore - " ".join(b_msg) + msg ) nextmsg = await self.client.get_messages( # type: ignore chat_id=self.message.chat.id, # type: ignore @@ -1352,15 +1359,14 @@ async def generate_sample_video(self, dl_path, gid, unwanted_files, ft_delete): if await aiopath.isfile(dl_path): if (await get_document_type(dl_path))[0]: checked = True - await cpu_eater_lock.acquire() - LOGGER.info(f"Creating Sample video: {self.name}") - res = await create_sample_video( - self, - dl_path, - sample_duration, - part_duration - ) - cpu_eater_lock.release() + async with cpu_eater_lock: + LOGGER.info(f"Creating Sample video: {self.name}") + res = await create_sample_video( + self, + dl_path, + sample_duration, + part_duration + ) if res: newfolder = ospath.splitext(dl_path)[0] name = dl_path.rsplit( diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 8b9bf6d8790f..f27f0475d8b0 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -370,6 +370,7 @@ async def retry_function(func, *args, **kwargs): **kwargs ) except: + await sleep(0.2) return await retry_function( func, *args, diff --git a/bot/helper/ext_utils/help_messages.py b/bot/helper/ext_utils/help_messages.py index ca39094d604e..f747c9750030 100644 --- a/bot/helper/ext_utils/help_messages.py +++ b/bot/helper/ext_utils/help_messages.py @@ -46,10 +46,27 @@ """ same_dir = """ -Multi links within the same upload directory only by replying to the first link/file: -sd +Move file(s)/folder(s) to new folder: -sd -/cmd -m 10(number of links/files) -sd folder name (multi message) -/cmd -b -sd folder name (bulk-message/file) +You can use this arg also to move multiple links/torrents contents to the same directory, so all links will be uploaded together as one task + +/cmd link -sd new folder (only one link inside new folder) +/cmd -m 10(number of links/files) -sd folder name (all links contents in one folder) +/cmd -b -sd folder name (reply to batch of message/file(each link on new line)) + +While using bulk you can also use this arg with different folder name along with the links in message or file batch +Example: +link1 -sd folder1 +link2 -sd folder1 +link3 -sd folder2 +link4 -sd folder2 +link5 -sd folder3 +link6 + +so link1 and link2 content will be uploaded from same folder which is folder1 +link3 and link4 content will be uploaded from same folder also which is folder2 +link5 will uploaded alone inside new folder named folder3 +link6 will get uploaded normally alone """ thumb = """ @@ -102,16 +119,20 @@ bulk = """ Bulk Download: -b -Bulk can be used by text message and by replying to a text file containing links separated by a new line. -You can use it only by replying to a message(text/file). +Bulk can be used only by replying to text message or text file contains links separated by new line. + Example: link1 -n new name -up remote1:path1 -rcf |key:value|key:value link2 -z -n new name -up remote2:path2 link3 -e -n new name -up remote2:path2 -Reply to this example by this cmd -> /cmd -b(bulk) or /cmd -b -sd folder name -You can set the start and end of the links from the bulk like seed, with -b start:end or only end by -b :end or only start by -b start. -The default start is from zero(first link) to infinity. -""" + +Reply to this example by this cmd -> /cmd -b(bulk) + +Note: Any arg along with the cmd will be setted to all links +/cmd -b -up remote: -z -sd folder name (all links contents in one zipped folder uploaded to one destination) +so you can't set different upload destinations along with link incase you have added -sd along with cmd +You can set start and end of the links from the bulk like seed, with -b start:end or only end by -b :end or only start by -b start. +The default start is from zero(first link) to inf.""" rlone_dl = """ Rclone Download: diff --git a/bot/helper/ext_utils/jdownloader_booter.py b/bot/helper/ext_utils/jdownloader_booter.py index 11e9e38aba0f..05f5ac498020 100644 --- a/bot/helper/ext_utils/jdownloader_booter.py +++ b/bot/helper/ext_utils/jdownloader_booter.py @@ -7,7 +7,7 @@ from aioshutil import rmtree from json import dump from random import randint -from asyncio import sleep +from asyncio import sleep, wait_for from re import match from bot import ( @@ -19,6 +19,7 @@ from .bot_utils import ( cmd_exec, new_task, + retry_function ) from myjd import MyJdApi from myjd.exception import ( @@ -169,5 +170,17 @@ async def connectToDevice(self): LOGGER.info("JDownloader is ready to use!") return True + async def check_jdownloader_state(self): + try: + await wait_for(retry_function(self.device.jd.version), timeout=10) + except: + is_connected = await self.jdconnect() + if not is_connected: + raise MYJDException(self.error) + await self.boot() + isDeviceConnected = await self.connectToDevice() + if not isDeviceConnected: + raise MYJDException(self.error) + jdownloader = JDownloader() diff --git a/bot/helper/ext_utils/media_utils.py b/bot/helper/ext_utils/media_utils.py index ab58b2cfb02b..336e517bf7c7 100644 --- a/bot/helper/ext_utils/media_utils.py +++ b/bot/helper/ext_utils/media_utils.py @@ -83,11 +83,15 @@ async def convert_video(listener, video_file, ext, retry=False): ] if listener.is_cancelled: return False - listener.suproc = await create_subprocess_exec( - *cmd, - stderr=PIPE - ) - _, stderr = await listener.suproc.communicate() + async with subprocess_lock: + listener.suproc = await create_subprocess_exec( + *cmd, + stderr=PIPE + ) + ( + _, + stderr + ) = await listener.suproc.communicate() if listener.is_cancelled: return False code = listener.suproc.returncode @@ -130,10 +134,11 @@ async def convert_audio(listener, audio_file, ext): ] if listener.is_cancelled: return False - listener.suproc = await create_subprocess_exec( - *cmd, - stderr=PIPE - ) + async with subprocess_lock: + listener.suproc = await create_subprocess_exec( + *cmd, + stderr=PIPE + ) ( _, stderr @@ -863,10 +868,11 @@ async def create_sample_video(listener, video_file, sample_duration, part_durati if listener.is_cancelled: return False - listener.suproc = await create_subprocess_exec( - *cmd, - stderr=PIPE - ) + async with subprocess_lock: + listener.suproc = await create_subprocess_exec( + *cmd, + stderr=PIPE + ) ( _, stderr @@ -897,7 +903,10 @@ async def edit_video_metadata(listener, dir): data = listener.metadata dir_path = Path(dir) - if not dir_path.lower().endswith(("mkv", "mp4")): + if not dir_path.suffix.lower().endswith(( + "mkv", + "mp4" + )): return dir file_name = dir_path.name @@ -1070,7 +1079,10 @@ async def add_attachment(listener, dir): data = listener.m_attachment dir_path = Path(dir) - if not dir_path.lower().endswith(("mkv", "mp4")): + if not dir_path.suffix.lower().endswith(( + "mkv", + "mp4" + )): return dir file_name = dir_path.name @@ -1078,9 +1090,11 @@ async def add_attachment(listener, dir): if data: if is_telegram_link(data): - msg = (await get_tg_link_message(data))[0] + msg = (await get_tg_link_message(data))[0] # type: ignore data = ( - await create_thumb(msg) if msg.photo or msg.document else "" + await create_thumb(msg) + if msg.photo or msg.document # type: ignore + else "" ) data_ext = data.split(".")[-1].lower() diff --git a/bot/helper/listeners/jdownloader_listener.py b/bot/helper/listeners/jdownloader_listener.py index 9211c0d84871..071c877f795a 100644 --- a/bot/helper/listeners/jdownloader_listener.py +++ b/bot/helper/listeners/jdownloader_listener.py @@ -17,21 +17,6 @@ from ..ext_utils.status_utils import get_task_by_gid -@new_task -async def update_download(gid, value): - try: - async with jd_lock: - del value["ids"][0] - new_gid = value["ids"][0] - jd_downloads[new_gid] = value - if task := await get_task_by_gid(f"{gid}"): - task._gid = new_gid - async with jd_lock: - del jd_downloads[gid] - except: - pass - - @new_task async def remove_download(gid): if intervals["stopAll"]: @@ -40,7 +25,7 @@ async def remove_download(gid): jdownloader.device.downloads.remove_links, # type: ignore package_ids=[gid], ) - if task := await get_task_by_gid(f"{gid}"): + if task := await get_task_by_gid(gid): await task.listener.on_download_error("Download removed manually!") async with jd_lock: del jd_downloads[gid] @@ -48,7 +33,7 @@ async def remove_download(gid): @new_task async def _on_download_complete(gid): - if task := await get_task_by_gid(f"{gid}"): + if task := await get_task_by_gid(gid): if task.listener.select: async with jd_lock: await retry_function( @@ -79,73 +64,64 @@ async def _jd_listener(): intervals["jd"] = "" break try: - await wait_for( - retry_function(jdownloader.device.jd.version), # type: ignore - timeout=10 - ) + await jdownloader.check_jdownloader_state() except: - is_connected = await jdownloader.jdconnect() - if not is_connected: - LOGGER.error(jdownloader.error) - continue - jdownloader.boot() # type: ignore - isDeviceConnected = await jdownloader.connectToDevice() - if not isDeviceConnected: - continue + continue try: packages = await jdownloader.device.downloads.query_packages( # type: ignore - [{"finished": True}] + [{ + "finished": True, + "saveTo": True + }] ) except: continue - finished = [ - pack["uuid"] - for pack - in packages - if pack.get( - "finished", - False - ) - ] all_packages = [ pack["uuid"] for pack in packages ] for ( - k, - v + d_gid, + d_dict ) in list(jd_downloads.items()): - if v["status"] == "down": - if k in all_packages: - for ( - index, - pid - ) in enumerate(v["ids"]): - if pid not in all_packages: - del jd_downloads[k]["ids"][index] - - else: - cdi = jd_downloads[k]["ids"] - if len(cdi) > 1: - await update_download(k, v) - else: - await remove_download(k) + if d_dict["status"] == "down": + for ( + index, + pid + ) in enumerate(d_dict["ids"]): + if pid not in all_packages: + del jd_downloads[d_gid]["ids"][index] + if len(jd_downloads[d_gid]["ids"]) == 0: + path = jd_downloads[d_gid]["path"] + jd_downloads[d_gid]["ids"] = [ + dl["uuid"] + for dl in all_packages + if dl["saveTo"].startswith(path) + ] + if len(jd_downloads[d_gid]["ids"]) == 0: + await remove_download(d_gid) - for gid in finished: - if ( - gid in jd_downloads and - jd_downloads[gid]["status"] == "down" - ): - is_finished = all( - did - in finished - for did - in jd_downloads[gid]["ids"] - ) - if is_finished: - jd_downloads[gid]["status"] = "done" - await _on_download_complete(gid) # type: ignore + if completed_packages := [ + pack["uuid"] + for pack in packages + if pack.get( + "finished", + False + ) + ]: + for ( + d_gid, + d_dict + ) in list(jd_downloads.items()): + if d_dict["status"] == "down": + is_finished = all( + did in completed_packages + for did in d_dict["ids"] + ) + if is_finished: + jd_downloads[d_gid]["status"] = "done" + await _on_download_complete(d_gid) async def on_download_start(): diff --git a/bot/helper/listeners/task_listener.py b/bot/helper/listeners/task_listener.py index aa8b95fb79b9..467e737703a9 100644 --- a/bot/helper/listeners/task_listener.py +++ b/bot/helper/listeners/task_listener.py @@ -14,6 +14,7 @@ from time import time from bot import ( + DOWNLOAD_DIR, LOGGER, aria2, config_dict, @@ -83,14 +84,15 @@ async def clean(self): except: pass - def remove_from_same_dir(self): - if ( - self.same_dir # type: ignore - and self.mid - in self.same_dir["tasks"] # type: ignore - ): - self.same_dir["tasks"].remove(self.mid) # type: ignore - self.same_dir["total"] -= 1 # type: ignore + async def remove_from_same_dir(self): + async with task_dict_lock: + if ( + self.folder_name + and self.same_dir # type: ignore + and self.mid in self.same_dir[self.folder_name]["tasks"] # type: ignore + ): + self.same_dir[self.folder_name]["tasks"].remove(self.mid) # type: ignore + self.same_dir[self.folder_name]["total"] -= 1 # type: ignore async def on_download_start(self): if ( @@ -114,6 +116,7 @@ async def on_download_start(self): ) async def on_download_complete(self): + await sleep(2) multi_links = False if ( config_dict["DATABASE_URL"] @@ -122,64 +125,54 @@ async def on_download_complete(self): ): await database.remove_download(self.raw_url) if ( - self.same_dir # type: ignore - and self.mid - in self.same_dir["tasks"] # type: ignore + self.folder_name + and self.same_dir # type: ignore + and self.mid in self.same_dir[self.folder_name]["tasks"] # type: ignore ): async with same_directory_lock: while True: async with task_dict_lock: - if self.mid not in self.same_dir["tasks"]: # type: ignore + if self.mid not in self.same_dir[self.folder_name]["tasks"]: # type: ignore return if ( - self.mid in self.same_dir["tasks"] and ( # type: ignore - self.same_dir["total"] == 1 # type: ignore - or len(self.same_dir["tasks"]) > 1 # type: ignore + self.mid in self.same_dir[self.folder_name]["tasks"] # type: ignore + and ( + self.same_dir[self.folder_name]["total"] <= 1 # type: ignore + or len(self.same_dir[self.folder_name]["tasks"]) > 1 # type: ignore ) ): + if self.same_dir[self.folder_name]["total"] > 1: # type: ignore + self.same_dir[self.folder_name]["tasks"].remove(self.mid) # type: ignore + self.same_dir[self.folder_name]["total"] -= 1 # type: ignore + spath = f"{self.dir}{self.folder_name}" + des_id = list(self.same_dir[self.folder_name]["tasks"])[0] # type: ignore + des_path = f"{DOWNLOAD_DIR}{des_id}{self.folder_name}" + await makedirs( + des_path, + exist_ok=True + ) + LOGGER.info(f"Moving files from {self.mid} to {des_id}") + for item in await listdir(spath): + if item.endswith(( + ".aria2", + ".!qB" + )): + continue + item_path = f"{self.dir}{self.folder_name}/{item}" + if item in await listdir(des_path): + await move( + item_path, + f"{des_path}/{self.mid}-{item}" + ) + else: + await move( + item_path, + f"{des_path}/{item}" + ) + multi_links = True break await sleep(1) - await sleep(2) # wait for qbitorrent or any other package to rearrange files and folders like removing !qB async with task_dict_lock: - if ( - self.same_dir # type: ignore - and self.same_dir["total"] > 1 # type: ignore - and self.mid - in self.same_dir["tasks"] # type: ignore - ): - self.same_dir["tasks"].remove(self.mid) # type: ignore - self.same_dir["total"] -= 1 # type: ignore - folder_name = self.same_dir["name"] # type: ignore - spath = f"{self.dir}{folder_name}" - des_path = f"{config_dict["DOWNLOAD_DIR"]}{list(self.same_dir["tasks"])[0]}{folder_name}" # type: ignore - await makedirs( - des_path, - exist_ok=True - ) - for item in await listdir(spath): - if item.endswith(( - ".aria2", - ".!qB" - )): - continue - item_path = f"{self.dir}{folder_name}/{item}" - if item in await listdir(des_path): - await move( - item_path, - f"{des_path}/{self.mid}-{item}" - ) - else: - await move( - item_path, - f"{des_path}/{item}" - ) - multi_links = True - elif ( - self.same_dir # type: ignore - and self.mid - not in self.same_dir["tasks"] # type: ignore - ): - return download = task_dict[self.mid] self.name = download.name() gid = download.gid() @@ -199,8 +192,8 @@ async def on_download_complete(self): await self.on_upload_error(f"{self.name} Downloaded!\n\nWaiting for other tasks to finish...") return - if self.same_dir: # type: ignore - self.name = self.same_dir["name"].split("/")[-1] # type: ignore + if self.folder_name: + self.name = self.folder_name.split("/")[-1] if not await aiopath.exists(f"{self.dir}/{self.name}"): try: @@ -727,7 +720,7 @@ async def on_download_error(self, error, button=None): if self.mid in task_dict: del task_dict[self.mid] count = len(task_dict) - self.remove_from_same_dir() + await self.remove_from_same_dir() msg = f"Sorry {self.tag}!\nYour download has been stopped." msg += f"\n\nReason : {escape(str(error))}" msg += f"\nPast : {get_readable_time(time() - self.time)}" diff --git a/bot/helper/listeners/ytdlp_listener.py b/bot/helper/listeners/ytdlp_listener.py index 7e57a3d43192..4987c13d3c8f 100644 --- a/bot/helper/listeners/ytdlp_listener.py +++ b/bot/helper/listeners/ytdlp_listener.py @@ -29,7 +29,6 @@ from yt_dlp import YoutubeDL -@new_task async def select_format(_, query, obj): data = query.data.split() message = query.message @@ -115,7 +114,7 @@ async def _event_handler(self): self.qual = None self.listener.is_cancelled = True await auto_delete_message( - self.listener.message, + None, self._reply_to ) self.event.set() diff --git a/bot/helper/task_utils/download_utils/direct_link_generator.py b/bot/helper/task_utils/download_utils/direct_link_generator.py index 1e65c7196fd8..760bb038c88d 100644 --- a/bot/helper/task_utils/download_utils/direct_link_generator.py +++ b/bot/helper/task_utils/download_utils/direct_link_generator.py @@ -42,15 +42,7 @@ def direct_link_generator(link): domain = urlparse(link).hostname if not domain: raise DirectDownloadLinkException("ERROR: Invalid URL") - if ( - "youtube.com" in domain - or "youtu.be" in domain - ): - raise DirectDownloadLinkException("ERROR: Use ytdl cmds for Youtube links") - elif ( - "yadi.sk" in link - or "disk.yandex." in link - ): + if "yadi.sk" in link or "disk.yandex." in link: return yandex_disk(link) elif "mediafire.com" in domain: return mediafire(link) @@ -278,12 +270,18 @@ def mediafire(url, session=None): else: _password = "" if final_link := findall( - r"https?:\/\/download\d+\.mediafire\.com\/\S+\/\S+\/\S+", - url + r"https?:\/\/download\d+\.mediafire\.com\/\S+\/\S+\/\S+", url ): return final_link[0] + def _repair_download(url, session): + try: + html = HTML(session.get(url).text) + if new_link := html.xpath('//a[@id="continue-btn"]/@href'): + return mediafire(f"https://mediafire.com/{new_link[0]}") + except Exception as e: + raise DirectDownloadLinkException(f"ERROR: {e.__class__.__name__}") from e if session is None: - session = Session() + session = create_scraper() parsed_url = urlparse(url) url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}" try: @@ -297,17 +295,23 @@ def mediafire(url, session=None): if html.xpath("//div[@class='passwordPrompt']"): if not _password: session.close() - raise DirectDownloadLinkException(f"ERROR: {PASSWORD_ERROR_MESSAGE}".format(url)) + raise DirectDownloadLinkException( + f"ERROR: {PASSWORD_ERROR_MESSAGE}".format(url) + ) try: - html = HTML(session.post(url, data={"downloadp": _password}).text) + html = HTML(session.post( + url, + data={"downloadp": _password}).text + ) except Exception as e: session.close() raise DirectDownloadLinkException(f"ERROR: {e.__class__.__name__}") from e if html.xpath("//div[@class='passwordPrompt']"): session.close() raise DirectDownloadLinkException("ERROR: Wrong password.") - if not (final_link := html.xpath("//a[@id='downloadButton']/@href")): - session.close() + if not (final_link := html.xpath('//a[@aria-label="Download file"]/@href')): + if repair_link := html.xpath("//a[@class='retry']/@href"): + return _repair_download(repair_link[0], session) raise DirectDownloadLinkException( "ERROR: No links found in this page Try Again" ) @@ -315,7 +319,10 @@ def mediafire(url, session=None): final_url = f"https://{final_link[0][2:]}" if _password: final_url += f"::{_password}" - return mediafire(final_url, session) + return mediafire( + final_url, + session + ) session.close() return final_link[0] @@ -1291,6 +1298,16 @@ def __get_info(folderkey): details["title"] = folder_infos[0]["name"] def __scraper(url): + session = create_scraper() + parsed_url = urlparse(url) + url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}" + def __repair_download(url): + try: + html = HTML(session.get(url).text) + if new_link := html.xpath('//a[@id="continue-btn"]/@href'): + return __scraper(f"https://mediafire.com/{new_link[0]}") + except: + return try: html = HTML(session.get(url).text) except: @@ -1306,8 +1323,12 @@ def __scraper(url): return if html.xpath("//div[@class='passwordPrompt']"): return - if final_link := html.xpath("//a[@id='downloadButton']/@href"): + if final_link := html.xpath('//a[@aria-label="Download file"]/@href'): + if final_link[0].startswith("//"): + return __scraper(f"https://{final_link[0][2:]}") return final_link[0] + if repair_link := html.xpath("//a[@class='retry']/@href"): + return __repair_download(repair_link[0]) def __get_content(folderKey, folderPath="", content_type="folders"): try: diff --git a/bot/helper/task_utils/download_utils/jd_download.py b/bot/helper/task_utils/download_utils/jd_download.py index 05e540a62b98..5e1c37b6ee98 100644 --- a/bot/helper/task_utils/download_utils/jd_download.py +++ b/bot/helper/task_utils/download_utils/jd_download.py @@ -10,9 +10,14 @@ ) from nekozee.handlers import CallbackQueryHandler from time import time -from aiofiles.os import path as aiopath +from aiofiles.os import ( + path as aiopath, + remove +) from aiofiles import open as aiopen from base64 import b64encode +from secrets import token_urlsafe +from myjd.exception import MYJDException from bot import ( LOGGER, @@ -34,9 +39,7 @@ stop_duplicate_check, ) from ...listeners.jdownloader_listener import on_download_start -from ...task_utils.status_utils.jdownloader_status import ( - JDownloaderStatus, -) +from ...task_utils.status_utils.jdownloader_status import JDownloaderStatus from ...task_utils.status_utils.queue_status import QueueStatus from ...telegram_helper.button_build import ButtonMaker from ...telegram_helper.message_utils import ( @@ -130,362 +133,385 @@ async def wait_for_configurations(self): await self._event_handler() if not self.listener.is_cancelled: await delete_message(self._reply_to) - return self.listener.is_cancelled + return not self.listener.is_cancelled -async def add_jd_download(listener, path): - async with jd_lock: - if jdownloader.device is None: - await listener.on_download_error(jdownloader.error) - return +async def get_online_packages(path, state="grabbing"): + if state == "grabbing": + queued_downloads = await retry_function( + jdownloader.device.linkgrabber.query_packages, [{"saveTo": True}] # type: ignore + ) + return [qd["uuid"] for qd in queued_downloads if qd["saveTo"].startswith(path)] + else: + download_packages = await retry_function( + jdownloader.device.downloads.query_packages, # type: ignore + [{"saveTo": True}], + ) + return [dl["uuid"] for dl in download_packages if dl["saveTo"].startswith(path)] - try: - await wait_for( - retry_function(jdownloader.device.jd.version), - timeout=10 - ) - except: - is_connected = await jdownloader.jdconnect() - if not is_connected: - await listener.on_download_error(jdownloader.error) - return - jdownloader.boot() # type: ignore - isDeviceConnected = await jdownloader.connectToDevice() - if not isDeviceConnected: - await listener.on_download_error(jdownloader.error) - return - if not jd_downloads: - await retry_function(jdownloader.device.linkgrabber.clear_list) - if odl := await retry_function( - jdownloader.device.downloads.query_packages, +def trim_path(path): + path_components = path.split("/") + + trimmed_components = [ + component[:255] if len(component) > 255 else component + for component in path_components + ] + + return "/".join(trimmed_components) + + +async def add_jd_download(listener, path): + try: + async with jd_lock: + gid = token_urlsafe(12) + jd_downloads[gid] = { + "status": "collect", + "path": path + } + if jdownloader.device is None: + raise MYJDException(jdownloader.error) + + await jdownloader.check_jdownloader_state() + + if not jd_downloads: + await retry_function(jdownloader.device.linkgrabber.clear_list) + if odl := await retry_function( + jdownloader.device.downloads.query_packages, + [{}] + ): + odl_list = [od["uuid"] for od in odl] + await retry_function( + jdownloader.device.downloads.remove_links, + package_ids=odl_list + ) + elif odl := await retry_function( + jdownloader.device.linkgrabber.query_packages, [{}] ): - odl_list = [ + if odl_list := [ od["uuid"] for od in odl - ] + if od.get( + "saveTo", + "" + ).startswith("/root/Downloads/") + ]: + await retry_function( + jdownloader.device.linkgrabber.remove_links, + package_ids=odl_list + ) + if await aiopath.exists(listener.link): + async with aiopen( + listener.link, + "rb" + ) as dlc: + content = await dlc.read() + content = b64encode(content) await retry_function( - jdownloader.device.downloads.remove_links, - package_ids=odl_list, + jdownloader.device.linkgrabber.add_container, + "DLC", + f";base64,{content.decode()}" ) - elif odl := await retry_function( - jdownloader.device.linkgrabber.query_packages, - [{}] - ): - if odl_list := [ - od["uuid"] - for od in odl - if od.get( - "saveTo", - "" - ).startswith("/root/Downloads/") - ]: + else: await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=odl_list, + jdownloader.device.linkgrabber.add_links, + [ + { + "autoExtract": False, + "links": listener.link, + "packageName": listener.name or None, + } + ] ) - if await aiopath.exists(listener.link): - async with aiopen( - listener.link, - "rb" - ) as dlc: - content = await dlc.read() - content = b64encode(content) - await retry_function( - jdownloader.device.linkgrabber.add_container, - "DLC", - f";base64,{content.decode()}", - ) - else: - await retry_function( - jdownloader.device.linkgrabber.add_links, - [ - { - "autoExtract": False, - "links": listener.link, - "packageName": listener.name or None, - } - ], - ) - - await sleep(0.5) - while await retry_function(jdownloader.device.linkgrabber.is_collecting): - pass - - start_time = time() - online_packages = [] - listener.size = 0 - corrupted_packages = [] - gid = 0 - remove_unknown = False - name = "" - error = "" - while (time() - start_time) < 60: - queued_downloads = await retry_function( - jdownloader.device.linkgrabber.query_packages, - [ - { - "bytesTotal": True, - "saveTo": True, - "availableOnlineCount": True, - "availableTempUnknownCount": True, - "availableUnknownCount": True, - } - ], - ) - if not online_packages and corrupted_packages and error: - await listener.on_download_error(error) - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=corrupted_packages, + await sleep(1) + while await retry_function(jdownloader.device.linkgrabber.is_collecting): + pass + start_time = time() + online_packages = [] + corrupted_packages = [] + remove_unknown = False + name = "" + error = "" + while (time() - start_time) < 60: + queued_downloads = await retry_function( + jdownloader.device.linkgrabber.query_packages, + [ + { + "bytesTotal": True, + "saveTo": True, + "availableOnlineCount": True, + "availableOfflineCount": True, + "availableTempUnknownCount": True, + "availableUnknownCount": True, + } + ], ) - return - for pack in queued_downloads: - online = pack.get( - "onlineCount", - 1 - ) - if online == 0: - error = f"{pack.get( - 'name', - '' - )}" - LOGGER.error(error) - corrupted_packages.append(pack["uuid"]) - continue - save_to = pack["saveTo"] - if gid == 0: - gid = pack["uuid"] - jd_downloads[gid] = {"status": "collect"} + if not online_packages and corrupted_packages and error: + await retry_function( + jdownloader.device.linkgrabber.remove_links, + package_ids=corrupted_packages + ) + raise MYJDException(error) + + for pack in queued_downloads: + if pack.get( + "onlineCount", + 1 + ) == 0: + error = f"{pack.get( + 'name', + '' + )}" + LOGGER.error(error) + corrupted_packages.append(pack["uuid"]) + continue + save_to = pack["saveTo"] + if not name: + if save_to.startswith("/root/Downloads/"): + name = save_to.replace( + "/root/Downloads/", + "", + 1 + ).split( + "/", + 1 + )[0] + else: + name = save_to.replace( + f"{path}/", + "", + 1 + ).split( + "/", + 1 + )[0] + name = name[:255] + + if ( + pack.get( + "tempUnknownCount", + 0 + ) > 0 + or pack.get( + "unknownCount", + 0 + ) > 0 + or pack.get( + "offlineCount", + 0 + ) > 0 + ): + remove_unknown = True + + listener.size += pack.get( + "bytesTotal", + 0 + ) + online_packages.append(pack["uuid"]) if save_to.startswith("/root/Downloads/"): - name = save_to.replace( - "/root/Downloads/", - "", - 1 - ).split( - "/", - 1 - )[0] - else: - name = save_to.replace( - f"{path}/", - "", - 1 - ).split( - "/", - 1 - )[0] - - if ( - pack.get("tempUnknownCount", 0) > 0 - or pack.get("unknownCount", 0) > 0 - ): - remove_unknown = True - - listener.size += pack.get( - "bytesTotal", - 0 + save_to = trim_path(save_to) + await retry_function( + jdownloader.device.linkgrabber.set_download_directory, + save_to.replace( + "/root/Downloads", + path, + 1 + ), + [pack["uuid"]], + ) + + if online_packages: + if listener.join and len(online_packages) > 1: + listener.name = listener.folder_name + await retry_function( + jdownloader.device.linkgrabber.move_to_new_package, + listener.name, + f"{path}/{listener.name}", + package_ids=online_packages, + ) + continue + break + else: + error = ( + name + or "Download Not Added! Maybe some issues in jdownloader or site!" ) - online_packages.append(pack["uuid"]) - if save_to.startswith("/root/Downloads/"): + if corrupted_packages or online_packages: + packages_to_remove = corrupted_packages + online_packages await retry_function( - jdownloader.device.linkgrabber.set_download_directory, - save_to.replace( - "/root/Downloads", - path, - 1 - ), - [pack["uuid"]], + jdownloader.device.linkgrabber.remove_links, + package_ids=packages_to_remove, ) + raise MYJDException(error) - if online_packages: - if listener.join and len(online_packages) > 1: - listener.name = listener.same_dir["name"] - await retry_function( - jdownloader.device.linkgrabber.move_to_new_package, - listener.name, - f"{path}/{listener.name}", - package_ids=online_packages, - ) - continue - break - else: - error = ( - name or "Download Not Added! Maybe some issues in jdownloader or site!" - ) - await listener.on_download_error(error) - if corrupted_packages or online_packages: - packages_to_remove = corrupted_packages + online_packages + jd_downloads[gid]["ids"] = online_packages + + corrupted_links = [] + if remove_unknown: + links = await retry_function( + jdownloader.device.linkgrabber.query_links, + [{ + "packageUUIDs": online_packages, + "availability": True + }], + ) + corrupted_links = [ + link["uuid"] + for link in links + if link["availability"].lower() != "online" + ] + if corrupted_packages or corrupted_links: await retry_function( jdownloader.device.linkgrabber.remove_links, - package_ids=packages_to_remove, + corrupted_links, + corrupted_packages, ) - del jd_downloads[gid] - return - jd_downloads[gid]["ids"] = online_packages + listener.name = listener.name or name - corrupted_links = [] - if remove_unknown: - links = await retry_function( - jdownloader.device.linkgrabber.query_links, - [ - { - "packageUUIDs": online_packages, - "availability": True - } - ], + ( + msg, + button + ) = await stop_duplicate_check(listener) + if msg: + await retry_function( + jdownloader.device.linkgrabber.remove_links, + package_ids=online_packages + ) + await listener.on_download_error( + msg, + button ) - corrupted_links = [ - link["uuid"] - for link in links - if link["availability"].lower() != "online" - ] - if corrupted_packages or corrupted_links: + async with jd_lock: + del jd_downloads[gid] + return + + limit_exceeded = await limit_checker( + listener, + is_jd=True + ) + if limit_exceeded: await retry_function( jdownloader.device.linkgrabber.remove_links, - corrupted_links, - corrupted_packages, + package_ids=online_packages + ) + LOGGER.info(f"JDownloader Limit Exceeded: {listener.name} | {listener.size}") + jdmsg = await listener.on_download_error(limit_exceeded) + async with jd_lock: + del jd_downloads[gid] + await delete_links(listener.message) + await auto_delete_message( + listener.message, + jdmsg ) + return - listener.name = listener.name or name + if listener.select: + if not await JDownloaderHelper(listener).wait_for_configurations(): + await retry_function( + jdownloader.device.linkgrabber.remove_links, + package_ids=online_packages, + ) + await listener.remove_from_same_dir() + async with jd_lock: + del jd_downloads[gid] + return + else: + online_packages = await get_online_packages(path) + if not online_packages: + raise MYJDException("This Download have been removed manually!") + async with jd_lock: + jd_downloads[gid]["ids"] = online_packages + + ( + add_to_queue, + event + ) = await check_running_tasks(listener) + if add_to_queue: + LOGGER.info(f"Added to Queue/Download: {listener.name}") + async with task_dict_lock: + task_dict[listener.mid] = QueueStatus( + listener, + gid, + "dl" + ) + await listener.on_download_start() + if listener.multi <= 1: + await send_status_message(listener.message) + await event.wait() # type: ignore + if listener.is_cancelled: + return + async with queue_dict_lock: + non_queued_dl.add(listener.mid) + + await jdownloader.check_jdownloader_state() + online_packages = await get_online_packages(path) + if not online_packages: + raise MYJDException("This Download have been removed manually!") + async with jd_lock: + jd_downloads[gid]["ids"] = online_packages - ( - msg, - button - ) = await stop_duplicate_check(listener) - if msg: await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=online_packages - ) - await listener.on_download_error( - msg, - button - ) - async with jd_lock: - del jd_downloads[gid] - return - if limit_exceeded := await limit_checker( - listener, - is_jd=True - ): - LOGGER.info( - f"JDownloader Limit Exceeded: {listener.name} | {listener.size}" - ) - jdmsg = await listener.on_download_error(limit_exceeded) - await delete_links(listener.message) - await auto_delete_message( - listener.message, - jdmsg + jdownloader.device.linkgrabber.move_to_downloadlist, + package_ids=online_packages, ) - return - if listener.select: - if await JDownloaderHelper(listener).wait_for_configurations(): + await sleep(1) + + online_packages = await get_online_packages( + path, + "down" + ) + if not online_packages: + online_packages = await get_online_packages(path) + if not online_packages: + raise MYJDException("This Download have been removed manually!") await retry_function( - jdownloader.device.linkgrabber.remove_links, + jdownloader.device.linkgrabber.move_to_downloadlist, package_ids=online_packages, ) - listener.remove_from_same_dir() - return - else: - queued_downloads = await retry_function( - jdownloader.device.linkgrabber.query_packages, - [{"saveTo": True}] + await sleep(1) + online_packages = await get_online_packages( + path, + "down" ) - updated_packages = [ - qd["uuid"] - for qd - in queued_downloads - if qd["saveTo"].startswith(path) - ] - async with jd_lock: - online_packages = [ - pack - for pack - in online_packages - if pack - in updated_packages - ] - if gid not in online_packages: - del jd_downloads[gid] - gid = online_packages[0] - jd_downloads[gid] = {"status": "collect"} - jd_downloads[gid]["ids"] = online_packages + if not online_packages: + raise MYJDException("This Download have been removed manually!") + + async with jd_lock: + jd_downloads[gid]["status"] = "down" + jd_downloads[gid]["ids"] = online_packages + + await retry_function( + jdownloader.device.downloads.force_download, + package_ids=online_packages + ) - ( - add_to_queue, - event - ) = await check_running_tasks(listener) - if add_to_queue: - LOGGER.info(f"Added to Queue/Download: {listener.name}") async with task_dict_lock: - task_dict[listener.mid] = QueueStatus( + task_dict[listener.mid] = JDownloaderStatus( listener, - f"{gid}", - "dl" + gid ) - await listener.on_download_start() - if listener.multi <= 1: - await send_status_message(listener.message) - await event.wait() # type: ignore + await on_download_start() - if listener.is_cancelled: - return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) - - await retry_function( - jdownloader.device.linkgrabber.move_to_downloadlist, - package_ids=online_packages, - ) - - await sleep(1) - - download_packages = await retry_function( - jdownloader.device.downloads.query_packages, - [{"saveTo": True}], - ) - async with jd_lock: - packages = [] - for pack in download_packages: - if pack["saveTo"].startswith(path): - if not packages: - del jd_downloads[gid] - gid = pack["uuid"] - jd_downloads[gid] = {"status": "down"} - packages.append(pack["uuid"]) - if packages: - jd_downloads[gid]["ids"] = packages - - if not packages: - await listener.on_download_error("This Download have been removed manually!") + if add_to_queue: + LOGGER.info(f"Start Queued Download from JDownloader: {listener.name}") + else: + LOGGER.info(f"Download with JDownloader: {listener.name}") + await listener.on_download_start() + if listener.multi <= 1: + await send_status_message(listener.message) + except ( + Exception, + MYJDException + ) as e: + await listener.on_download_error(f"{e}".strip()) async with jd_lock: del jd_downloads[gid] - return - - await retry_function( - jdownloader.device.downloads.force_download, - package_ids=packages, - ) - - async with task_dict_lock: - task_dict[listener.mid] = JDownloaderStatus( - listener, - f"{gid}", - ) - - await on_download_start() - - if add_to_queue: - LOGGER.info(f"Start Queued Download from JDownloader: {listener.name}") - else: - LOGGER.info(f"Download with JDownloader: {listener.name}") - await listener.on_download_start() - if listener.multi <= 1: - await send_status_message(listener.message) + finally: + if await aiopath.exists(listener.link): + await remove(listener.link) diff --git a/bot/helper/task_utils/status_utils/jdownloader_status.py b/bot/helper/task_utils/status_utils/jdownloader_status.py index 06cb4b6239f9..19b9b99a102a 100644 --- a/bot/helper/task_utils/status_utils/jdownloader_status.py +++ b/bot/helper/task_utils/status_utils/jdownloader_status.py @@ -112,7 +112,7 @@ def _eng_ver(self): async def _update(self): self._info = await get_download( - int(self._gid), + self._gid, self._info ) @@ -183,8 +183,8 @@ async def cancel_task(self): LOGGER.info(f"Cancelling Download: {self.name()}") await retry_function( jdownloader.device.downloads.remove_links, # type: ignore - package_ids=jd_downloads[int(self._gid)]["ids"], + package_ids=jd_downloads[self._gid]["ids"], ) async with jd_lock: - del jd_downloads[int(self._gid)] + del jd_downloads[self._gid] await self.listener.on_download_error("Download cancelled by user!") diff --git a/bot/helper/task_utils/status_utils/media_convert_status.py b/bot/helper/task_utils/status_utils/media_convert_status.py index a0b52ba0d710..f99bbf3b0706 100644 --- a/bot/helper/task_utils/status_utils/media_convert_status.py +++ b/bot/helper/task_utils/status_utils/media_convert_status.py @@ -1,5 +1,8 @@ from time import time -from bot import LOGGER +from bot import ( + LOGGER, + subprocess_lock +) from ...ext_utils.status_utils import ( get_readable_file_size, get_readable_time, @@ -83,6 +86,7 @@ def task(self): async def cancel_task(self): LOGGER.info(f"Cancelling Converting: {self.listener.name}") self.listener.is_cancelled = True - if self.listener.suproc is not None and self.listener.suproc.returncode is None: - self.listener.suproc.kill() + async with subprocess_lock: + if self.listener.suproc is not None and self.listener.suproc.returncode is None: + self.listener.suproc.kill() await self.listener.on_upload_error("Converting stopped by user!") diff --git a/bot/helper/task_utils/status_utils/sample_video_status.py b/bot/helper/task_utils/status_utils/sample_video_status.py index 42940a017f05..fff009adbc22 100644 --- a/bot/helper/task_utils/status_utils/sample_video_status.py +++ b/bot/helper/task_utils/status_utils/sample_video_status.py @@ -1,4 +1,7 @@ -from bot import LOGGER +from bot import ( + LOGGER, + subprocess_lock +) from ...ext_utils.status_utils import ( get_readable_file_size, MirrorStatus, @@ -91,9 +94,10 @@ def task(self): async def cancel_task(self): LOGGER.info(f"Cancelling Sample Video: {self.listener.name}") self.listener.is_cancelled = True - if ( - self.listener.suproc is not None - and self.listener.suproc.returncode is None - ): - self.listener.suproc.kill() + async with subprocess_lock: + if ( + self.listener.suproc is not None + and self.listener.suproc.returncode is None + ): + self.listener.suproc.kill() await self.listener.on_upload_error("Creating sample video stopped by user!") diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index 2ec17514a755..a1fbc4474c52 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -22,6 +22,7 @@ from bot import ( LOGGER, bot, + bot_loop, bot_name, cached_dict, config_dict, @@ -135,13 +136,13 @@ async def auto_delete_message( if (config_dict["DELETE_LINKS"] and int(config_dict["AUTO_DELETE_MESSAGE_DURATION"]) ) > 0: - async def delete_delay(): + async def auto_delete(): await sleep(config_dict["AUTO_DELETE_MESSAGE_DURATION"]) if cmd_message is not None: await delete_message(cmd_message) if bot_message is not None: await delete_message(bot_message) - create_task(delete_delay()) + bot_loop.create_task(auto_delete()) async def delete_links(message): diff --git a/bot/modules/bot_settings.py b/bot/modules/bot_settings.py index 0a97020bd29c..5097fdd56c2d 100644 --- a/bot/modules/bot_settings.py +++ b/bot/modules/bot_settings.py @@ -46,6 +46,7 @@ index_urls, intervals, jd_downloads, + jd_lock, nzb_options, qbit_options, qbittorrent_client, @@ -688,7 +689,7 @@ async def edit_variable(message, pre_message, key): "JD_EMAIL", "JD_PASS" ]: - jdownloader.initiate() # type: ignore + await jdownloader.initiate() elif key == "RSS_DELAY": add_job() elif key == "USET_SERVERS": @@ -864,19 +865,27 @@ async def edit_nzb_server(message, pre_message, key, index=0): async def sync_jdownloader(): - if ( - not config_dict["DATABASE_URL"] - or jdownloader.device is None - ): - return - try: - await wait_for( - retry_function( - jdownloader.update_devices - ), - timeout=10 - ) - except: + async with jd_lock: + if ( + not config_dict["DATABASE_URL"] + or jdownloader.device is None + ): + return + try: + await wait_for( + retry_function(jdownloader.update_devices), + timeout=10 + ) + except: + is_connected = await jdownloader.jdconnect() + if not is_connected: + LOGGER.error(jdownloader.error) + return + isDeviceConnected = await jdownloader.connectToDevice() + if not isDeviceConnected: + LOGGER.error(jdownloader.error) + return + await jdownloader.device.system.exit_jd() is_connected = await jdownloader.jdconnect() if not is_connected: LOGGER.error(jdownloader.error) @@ -884,17 +893,8 @@ async def sync_jdownloader(): isDeviceConnected = await jdownloader.connectToDevice() if not isDeviceConnected: LOGGER.error(jdownloader.error) - return - await jdownloader.device.system.exit_jd() if await aiopath.exists("cfg.zip"): await remove("cfg.zip") - is_connected = await jdownloader.jdconnect() - if not is_connected: - LOGGER.error(jdownloader.error) - return - isDeviceConnected = await jdownloader.connectToDevice() - if not isDeviceConnected: - LOGGER.error(jdownloader.error) await ( await create_subprocess_exec( "7z", @@ -1146,14 +1146,8 @@ async def edit_bot_settings(client, query): show_alert=True, ) return - if jd_downloads: - await query.answer( - "You can't sync settings while using jdownloader!", - show_alert=True, - ) - return await query.answer( - "Syncronization Started. JDownloader will get restarted. It takes up to 5 sec!", + "Syncronization Started. JDownloader will get restarted. It takes up to 10 sec!", show_alert=True, ) await sync_jdownloader() diff --git a/bot/modules/clone.py b/bot/modules/clone.py index fa0272c5dd07..41686b445971 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -142,7 +142,6 @@ async def new_event(self): await self.run_multi( input_list, - "", Clone ) diff --git a/bot/modules/mirror_leech.py b/bot/modules/mirror_leech.py index 88905cdf3b1e..67de7fa69056 100644 --- a/bot/modules/mirror_leech.py +++ b/bot/modules/mirror_leech.py @@ -1,7 +1,4 @@ -from aiofiles.os import ( - path as aiopath, - remove -) +from aiofiles.os import path as aiopath from base64 import b64encode from re import match as re_match @@ -12,7 +9,8 @@ DOWNLOAD_DIR, LOGGER, bot, - bot_loop + bot_loop, + task_dict_lock ) from ..helper.ext_utils.bot_utils import ( COMMAND_USAGE, @@ -50,7 +48,6 @@ send_message, edit_message, ) -from myjd.exception import MYJDException class Mirror(TaskListener): @@ -130,12 +127,12 @@ async def new_event(self): "-ap": "", "-authpass": "", "-h": "", "-headers": "", "-t": "", "-thumb": "", + "-tl": "", "-thumblayout": "", "-ca": "", "-convertaudio": "", "-cv": "", "-convertvideo": "", "-ns": "", "-namesub": "", "-md": "", "-metadata": "", - "-mda": "", "-metaattachment": "", - "-tl": "", "-thumblayout": "", + "-mda": "", "-metaattachment": "" } arg_parser( @@ -168,10 +165,16 @@ async def new_event(self): self.thumbnail_layout = args["-tl"] or args["-thumblayout"] self.as_doc = args["-doc"] or args["-document"] self.as_med = args["-med"] or args["-media"] + self.folder_name = (( + f"/{args["-sd"]}" or + f"/{args["-samedir"]}" + ) if ( + len(args["-sd"]) or + len(args["-samedir"]) + ) > 0 else "") headers = args["-h"] or args["-headers"] is_bulk = args["-b"] or args["-bulk"] - folder_name = args["-sd"] or args["-samedir"] bulk_start = 0 bulk_end = 0 @@ -215,21 +218,28 @@ async def new_event(self): is_bulk = True if not is_bulk: - if folder_name: - self.seed = False - ratio = None - seed_time = None - folder_name = f"/{folder_name}" - if not self.same_dir: - self.same_dir = { - "total": self.multi, - "tasks": set(), - "name": folder_name, - } - self.same_dir["tasks"].add(self.mid) - elif self.same_dir: - self.same_dir["total"] -= 1 - + if self.multi > 0: + if self.folder_name: + self.seed = False + ratio = None + seed_time = None + async with task_dict_lock: + if self.folder_name in self.same_dir: + self.same_dir[self.folder_name]["tasks"].add(self.mid) + for fd_name in self.same_dir: + if fd_name != self.folder_name: + self.same_dir[fd_name]["total"] -= 1 + elif self.same_dir: + self.same_dir[self.folder_name] = {"total": self.multi, "tasks": {self.mid}} + for fd_name in self.same_dir: + if fd_name != self.folder_name: + self.same_dir[fd_name]["total"] -= 1 + else: + self.same_dir = {self.folder_name: {"total": self.multi, "tasks": {self.mid}}} + elif self.same_dir: + async with task_dict_lock: + for fd_name in self.same_dir: + self.same_dir[fd_name]["total"] -= 1 else: await delete_message(self.pmsg) await self.init_bulk( @@ -244,12 +254,13 @@ async def new_event(self): del self.bulk[0] await self.run_multi( - input_list, - folder_name, - Mirror - ) + input_list, + Mirror + ) - path = f"{DOWNLOAD_DIR}{self.mid}{folder_name}" + await self.get_tag(text) + + path = f"{DOWNLOAD_DIR}{self.mid}{self.folder_name}" if ( not self.link @@ -282,13 +293,12 @@ async def new_event(self): self.message, tmsg ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await delete_message(self.pmsg) return if isinstance(reply_to, list): self.bulk = reply_to - self.same_dir = {} b_msg = input_list[:1] self.options = " ".join(input_list[1:]) b_msg.append(f"{self.bulk[0]} -m {len(self.bulk)} {self.options}") @@ -371,7 +381,7 @@ async def new_event(self): COMMAND_USAGE["mirror"][0], COMMAND_USAGE["mirror"][1] ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await delete_message(self.pmsg) await auto_delete_message( self.message, @@ -393,7 +403,7 @@ async def new_event(self): self.message, e ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await delete_message(self.pmsg) await auto_delete_message( self.message, @@ -442,7 +452,7 @@ async def new_event(self): self.message, e ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await auto_delete_message( self.message, dmsg @@ -464,28 +474,10 @@ async def new_event(self): path ) elif self.is_jd: - try: - await add_jd_download( - self, - path - ) - except ( - Exception, - MYJDException - ) as e: - jmsg = await send_message( - self.message, - f"{e}".strip() - ) - self.remove_from_same_dir() - await auto_delete_message( - self.message, - jmsg - ) - return - finally: - if await aiopath.exists(str(self.link)): - await remove(str(self.link)) + await add_jd_download( + self, + path + ) elif self.is_qbit: await add_qb_torrent( self, diff --git a/bot/modules/ytdlp.py b/bot/modules/ytdlp.py index c8e301b14aa4..41008f3460ca 100644 --- a/bot/modules/ytdlp.py +++ b/bot/modules/ytdlp.py @@ -3,7 +3,8 @@ LOGGER, bot, bot_loop, - config_dict + config_dict, + task_dict_lock ) from bot.helper.ext_utils.bot_utils import ( COMMAND_USAGE, @@ -68,30 +69,30 @@ async def new_event(self): qual = "" args = { - "-doc": False, - "-med": False, - "-s": False, - "-b": False, - "-z": False, - "-sv": False, - "-ss": False, - "-f": False, - "-fd": False, - "-fu": False, - "-ml": False, + "-doc": False, "-document": False, + "-med": False, "-media": False, + "-s": False, "-select": False, + "-b": False, "-bulk": False, + "-z": False, "-zip": False, "-compress": False, + "-sv": False, "-samplevideo": False, + "-ss": False, "-screenshot": False, + "-f": False, "-forcerun": False, + "-fd": False, "-forcedownload": False, + "-fu": False, "-forceupload": False, + "-ml": False, "-mixedleech": False, "-m": 0, - "-sp": 0, + "-sp": 0, "-splitsize": 0, "link": "", - "-sd": "", + "-sd": "", "-samedir": "", "-opt": "", - "-n": "", - "-up": "", + "-n": "", "-rename": "", + "-up": "", "-upload": "", "-rcf": "", - "-t": "", - "-ca": "", - "-cv": "", - "-ns": "", - "-tl": "", + "-t": "", "-thumb": "", + "-tl": "", "-thumblayout": "", + "-ca": "", "-convertaudio": "", + "-cv": "", "-convertvideo": "", + "-ns": "", "-namesub": "" } arg_parser( @@ -104,29 +105,35 @@ async def new_event(self): except: self.multi = 0 - self.select = args["-s"] - self.name = args["-n"] - self.up_dest = args["-up"] + self.select = args["-s"] or args["-select"] + self.name = args["-n"] or args["-rename"] + self.up_dest = args["-up"] or args["-upload"] self.rc_flags = args["-rcf"] self.link = args["link"] - self.compress = args["-z"] - self.thumb = args["-t"] - self.split_size = args["-sp"] - self.sample_video = args["-sv"] - self.screen_shots = args["-ss"] - self.force_run = args["-f"] - self.force_download = args["-fd"] - self.force_upload = args["-fu"] - self.convert_audio = args["-ca"] - self.convert_video = args["-cv"] - self.name_sub = args["-ns"] - self.mixed_leech = args["-ml"] - self.thumbnail_layout = args["-tl"] - self.as_doc = args["-doc"] - self.as_med = args["-med"] + self.compress = args["-z"] or args["-zip"] or args["-compress"] + self.thumb = args["-t"] or args["-thumb"] + self.thumbnail_layout = args["-tl"] or args["-thumblayout"] + self.split_size = args["-sp"] or args["-splitsize"] + self.sample_video = args["-sv"] or args["-samplevideo"] + self.screen_shots = args["-ss"] or args["-screenshot"] + self.force_run = args["-f"] or args["-forcerun"] + self.force_download = args["-fd"] or args["-forcedownload"] + self.force_upload = args["-fu"] or args["-forceupload"] + self.convert_audio = args["-ca"] or args["-convertaudio"] + self.convert_video = args["-cv"] or args["-convertvideo"] + self.name_sub = args["-ns"] or args["-namesub"] + self.mixed_leech = args["-ml"] or args["-mixedleech"] + self.as_doc = args["-doc"] or args["-document"] + self.as_med = args["-med"] or args["-media"] + self.folder_name = (( + f"/{args["-sd"]}" or + f"/{args["-samedir"]}" + ) if ( + len(args["-sd"]) or + len(args["-samedir"]) + ) > 0 else "") is_bulk = args["-b"] - folder_name = args["-sd"] bulk_start = 0 bulk_end = 0 @@ -144,17 +151,33 @@ async def new_event(self): is_bulk = True if not is_bulk: - if folder_name: - folder_name = f"/{folder_name}" - if not self.same_dir: - self.same_dir = { - "total": self.multi, - "tasks": set(), - "name": folder_name, - } - self.same_dir["tasks"].add(self.mid) - elif self.same_dir: - self.same_dir["total"] -= 1 + if self.multi > 0: + if self.folder_name: + async with task_dict_lock: + if self.folder_name in self.same_dir: + self.same_dir[self.folder_name]["tasks"].add(self.mid) + for fd_name in self.same_dir: + if fd_name != self.folder_name: + self.same_dir[fd_name]["total"] -= 1 + elif self.same_dir: + self.same_dir[self.folder_name] = { + "total": self.multi, + "tasks": {self.mid}, + } + for fd_name in self.same_dir: + if fd_name != self.folder_name: + self.same_dir[fd_name]["total"] -= 1 + else: + self.same_dir = { + self.folder_name: { + "total": self.multi, + "tasks": {self.mid}, + } + } + elif self.same_dir: + async with task_dict_lock: + for fd_name in self.same_dir: + self.same_dir[fd_name]["total"] -= 1 else: await delete_message(self.pmsg) await self.init_bulk( @@ -168,7 +191,7 @@ async def new_event(self): if len(self.bulk) != 0: del self.bulk[0] - path = f"{DOWNLOAD_DIR}{self.mid}{folder_name}" + path = f"{DOWNLOAD_DIR}{self.mid}{self.folder_name}" await self.get_tag(text) @@ -193,7 +216,7 @@ async def new_event(self): COMMAND_USAGE["yt"][0], COMMAND_USAGE["yt"][1] ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await delete_message(self.pmsg) await auto_delete_message( self.message, @@ -221,7 +244,7 @@ async def new_event(self): self.message, e ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await auto_delete_message( self.message, emsg @@ -307,7 +330,7 @@ async def new_event(self): self.message, f"{self.tag} {msg}" ) - self.remove_from_same_dir() + await self.remove_from_same_dir() await auto_delete_message( self.message, emsg @@ -316,14 +339,13 @@ async def new_event(self): finally: await self.run_multi( input_list, - folder_name, YtDlp ) if not qual: qual = await YtSelection(self).get_quality(result) if qual is None: - self.remove_from_same_dir() + await self.remove_from_same_dir() return await delete_message(self.pmsg)