From a42d1b3d5014a507718823b8342bde6ffdabd222 Mon Sep 17 00:00:00 2001 From: Dawn India Date: Wed, 30 Oct 2024 17:14:31 +0530 Subject: [PATCH] Minor fixes and improvements. N/A --- bot/__main__.py | 2 +- bot/helper/common.py | 6 + bot/helper/ext_utils/bot_utils.py | 15 - bot/helper/ext_utils/jdownloader_booter.py | 135 +-- bot/helper/ext_utils/task_manager.py | 9 +- bot/helper/listeners/aria2_listener.py | 2 - bot/helper/listeners/jdownloader_listener.py | 24 +- bot/helper/listeners/nzb_listener.py | 1 - bot/helper/listeners/qbit_listener.py | 10 +- bot/helper/listeners/task_listener.py | 2 - .../download_utils/aria2_download.py | 4 - .../download_utils/direct_downloader.py | 6 +- .../task_utils/download_utils/gd_download.py | 4 - .../task_utils/download_utils/jd_download.py | 209 ++--- .../download_utils/nzb_downloader.py | 4 - .../download_utils/qbit_download.py | 34 +- .../download_utils/rclone_download.py | 4 - .../download_utils/telegram_download.py | 4 - .../download_utils/yt_dlp_download.py | 12 +- .../status_utils/jdownloader_status.py | 10 +- bot/modules/bot_settings.py | 40 +- bot/modules/clone.py | 18 +- bot/modules/rss.py | 20 +- config_sample.env | 5 +- myjd/myjdapi.py | 884 ++---------------- requirements.txt | 1 - 26 files changed, 270 insertions(+), 1195 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index 838b6c2d9572..9fdc0b1e098b 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -301,7 +301,7 @@ async def main(): if config_dict["DATABASE_URL"]: await database.db_load() await gather( - jdownloader.initiate(), + jdownloader.boot(), sync_to_async(clean_all), bot_settings.initiate_search_tools(), restart_notification(), diff --git a/bot/helper/common.py b/bot/helper/common.py index 7dd159e67eeb..00ed4536a629 100644 --- a/bot/helper/common.py +++ b/bot/helper/common.py @@ -1761,6 +1761,9 @@ async def substitute(self, dl_path): if sen else 0 ) + if len(name.encode()) > 255: + LOGGER.error(f"Substitute: {name} is too long") + return dl_path new_path = ospath.join( up_dir, name @@ -1806,6 +1809,9 @@ async def substitute(self, dl_path): if sen else 0 ) + if len(file_.encode()) > 255: + LOGGER.error(f"Substitute: {file_} is too long") + continue await move( f_path, ospath.join( diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 65e7d9b6d328..1487a48b6d6d 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -371,21 +371,6 @@ def update_user_ldata(id_, key, value): user_data[id_][key] = value -async def retry_function(func, *args, **kwargs): - try: - return await func( - *args, - **kwargs - ) - except: - await sleep(0.2) - return await retry_function( - func, - *args, - **kwargs - ) - - async def cmd_exec(cmd, shell=False): if shell: proc = await create_subprocess_shell( diff --git a/bot/helper/ext_utils/jdownloader_booter.py b/bot/helper/ext_utils/jdownloader_booter.py index 05f5ac498020..5eae99d623c3 100644 --- a/bot/helper/ext_utils/jdownloader_booter.py +++ b/bot/helper/ext_utils/jdownloader_booter.py @@ -7,28 +7,17 @@ from aioshutil import rmtree from json import dump from random import randint -from asyncio import sleep, wait_for from re import match from bot import ( - config_dict, - LOGGER, - jd_lock, - bot_name + bot_name, + config_dict ) from .bot_utils import ( cmd_exec, - new_task, - retry_function + new_task ) from myjd import MyJdApi -from myjd.exception import ( - MYJDException, - MYJDAuthFailedException, - MYJDEmailForbiddenException, - MYJDEmailInvalidException, - MYJDErrorEmailNotConfirmedException, -) class JDownloader(MyJdApi): @@ -37,18 +26,8 @@ def __init__(self): self._username = "" self._password = "" self._device_name = "" + self.is_connected = False self.error = "JDownloader Credentials not provided!" - self.device = None - self.set_app_key("zee") - - @new_task - async def initiate(self): - self.device = None - async with jd_lock: - is_connected = await self.jdconnect() - if is_connected: - await self.boot() - await self.connectToDevice() @new_task async def boot(self): @@ -58,7 +37,13 @@ async def boot(self): "-f", "java" ]) - self.device = None + if ( + not config_dict["JD_EMAIL"] or + not config_dict["JD_PASS"] + ): + self.is_connected = False + self.error = "JDownloader Credentials not provided!" + return self.error = "Connecting... Try agin after couple of seconds" self._device_name = f"{randint(0, 1000)}@{bot_name}" jdata = { @@ -67,6 +52,20 @@ async def boot(self): "devicename": f"{self._device_name}", "email": config_dict["JD_EMAIL"], } + remote_data = { + "localapiserverheaderaccesscontrollalloworigin": "", + "deprecatedapiport": 3128, + "localapiserverheaderxcontenttypeoptions": "nosniff", + "localapiserverheaderxframeoptions": "DENY", + "externinterfaceenabled": True, + "deprecatedapilocalhostonly": True, + "localapiserverheaderreferrerpolicy": "no-referrer", + "deprecatedapienabled": True, + "localapiserverheadercontentsecuritypolicy": "default-src 'self'", + "jdanywhereapienabled": True, + "externinterfacelocalhostonly": False, + "localapiserverheaderxxssprotection": "1; mode=block", + } await makedirs( "/JDownloader/cfg", exist_ok=True @@ -80,6 +79,12 @@ async def boot(self): jdata, sf ) + with open( + "/JDownloader/cfg/org.jdownloader.api.RemoteAPIConfig.json", + "w", + ) as rf: + rf.truncate(0) + dump(remote_data, rf) if not await path.exists("/JDownloader/JDownloader.jar"): pattern = r"JDownloader\.jar\.backup.\d$" for filename in await listdir("/JDownloader"): @@ -95,6 +100,7 @@ async def boot(self): await rmtree("/JDownloader/update") await rmtree("/JDownloader/tmp") cmd = "java -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 -Djava.awt.headless=true -jar /JDownloader/JDownloader.jar" + self.is_connected = True ( _, __, @@ -103,84 +109,9 @@ async def boot(self): cmd, shell=True ) + self.is_connected = False if code != -9: await self.boot() - async def jdconnect(self): - if ( - not config_dict["JD_EMAIL"] - or not config_dict["JD_PASS"] - ): - return False - try: - await self.connect( - config_dict["JD_EMAIL"], - config_dict["JD_PASS"] - ) - return True - except ( - MYJDAuthFailedException, - MYJDEmailForbiddenException, - MYJDEmailInvalidException, - MYJDErrorEmailNotConfirmedException, - ) as err: - self.error = f"{err}".strip() - LOGGER.info(f"Failed to connect with jdownloader! ERROR: {self.error}") - self.device = None - return False - except MYJDException as e: - self.error = f"{e}".strip() - LOGGER.info( - f"Failed to connect with jdownloader! Retrying... ERROR: {self.error}" - ) - return await self.jdconnect() - - async def connectToDevice(self): - self.error = "Connecting to device..." - await sleep(0.5) - while True: - self.device = None - if ( - not config_dict["JD_EMAIL"] - or not config_dict["JD_PASS"] - ): - self.error = "JDownloader Credentials not provided!" - await cmd_exec([ - "pkill", - "-9", - "-f", - "java" - ]) - return False - try: - await self.update_devices() - if not (devices := self.list_devices()): - continue - for device in devices: - if self._device_name == device["name"]: - self.device = self.get_device(f"{self._device_name}") - break - else: - continue - except: - continue - break - await self.device.enable_direct_connection() - self.error = "" - LOGGER.info("JDownloader is ready to use!") - return True - - async def check_jdownloader_state(self): - try: - await wait_for(retry_function(self.device.jd.version), timeout=10) - except: - is_connected = await self.jdconnect() - if not is_connected: - raise MYJDException(self.error) - await self.boot() - isDeviceConnected = await self.connectToDevice() - if not isDeviceConnected: - raise MYJDException(self.error) - jdownloader = JDownloader() diff --git a/bot/helper/ext_utils/task_manager.py b/bot/helper/ext_utils/task_manager.py index 666b8eb211c8..2de0b86e1df8 100644 --- a/bot/helper/ext_utils/task_manager.py +++ b/bot/helper/ext_utils/task_manager.py @@ -1,7 +1,4 @@ -from asyncio import ( - Event, - sleep -) +from asyncio import Event from bot import ( config_dict, @@ -148,13 +145,13 @@ async def check_running_tasks(listener, state="dl"): async def start_dl_from_queued(mid: int): queued_dl[mid].set() del queued_dl[mid] - await sleep(0.7) + non_queued_dl.add(mid) async def start_up_from_queued(mid: int): queued_up[mid].set() del queued_up[mid] - await sleep(0.7) + non_queued_up.add(mid) async def start_from_queued(): diff --git a/bot/helper/listeners/aria2_listener.py b/bot/helper/listeners/aria2_listener.py index 3a855de1ef0d..2ffa6b4f0a44 100644 --- a/bot/helper/listeners/aria2_listener.py +++ b/bot/helper/listeners/aria2_listener.py @@ -30,8 +30,6 @@ ) from ..task_utils.status_utils.aria2_status import Aria2Status from ..telegram_helper.message_utils import ( - auto_delete_message, - delete_links, delete_message, send_message, update_status_message, diff --git a/bot/helper/listeners/jdownloader_listener.py b/bot/helper/listeners/jdownloader_listener.py index 7646f9a0f919..57f1d826aa2c 100644 --- a/bot/helper/listeners/jdownloader_listener.py +++ b/bot/helper/listeners/jdownloader_listener.py @@ -5,10 +5,7 @@ jd_downloads, intervals ) -from ..ext_utils.bot_utils import ( - new_task, - retry_function -) +from ..ext_utils.bot_utils import new_task from ..ext_utils.jdownloader_booter import jdownloader from ..ext_utils.status_utils import get_task_by_gid @@ -17,9 +14,8 @@ async def remove_download(gid): if intervals["stopAll"]: return - await retry_function( - jdownloader.device.downloads.remove_links, # type: ignore - package_ids=jd_downloads[gid]["ids"], + await jdownloader.device.downloads.remove_links( + package_ids=jd_downloads[gid]["ids"] ) if task := await get_task_by_gid(gid): await task.listener.on_download_error("Download removed manually!") @@ -32,8 +28,7 @@ async def _on_download_complete(gid): if task := await get_task_by_gid(gid): if task.listener.select: async with jd_lock: - await retry_function( - jdownloader.device.downloads.cleanup, # type: ignore + await jdownloader.device.downloads.cleanup( "DELETE_DISABLED", "REMOVE_LINKS_AND_DELETE_FILES", "SELECTED", @@ -44,8 +39,7 @@ async def _on_download_complete(gid): return async with jd_lock: if gid in jd_downloads: - await retry_function( - jdownloader.device.downloads.remove_links, # type: ignore + await jdownloader.device.downloads.remove_links( package_ids=jd_downloads[gid]["ids"], ) del jd_downloads[gid] @@ -60,11 +54,7 @@ async def _jd_listener(): intervals["jd"] = "" break try: - await jdownloader.check_jdownloader_state() - except: - continue - try: - packages = await jdownloader.device.downloads.query_packages( # type: ignore + packages = await jdownloader.device.downloads.query_packages( [{ "finished": True, "saveTo": True @@ -78,8 +68,6 @@ async def _jd_listener(): for pack in packages } - if not all_packages: - continue for ( d_gid, d_dict diff --git a/bot/helper/listeners/nzb_listener.py b/bot/helper/listeners/nzb_listener.py index afd7a0b2f0ef..bcf2f776a5d7 100644 --- a/bot/helper/listeners/nzb_listener.py +++ b/bot/helper/listeners/nzb_listener.py @@ -20,7 +20,6 @@ limit_checker, stop_duplicate_check ) -from ..telegram_helper.message_utils import auto_delete_message async def _remove_job(nzo_id, mid): diff --git a/bot/helper/listeners/qbit_listener.py b/bot/helper/listeners/qbit_listener.py index d4b17a3a6b88..f9738df2447f 100644 --- a/bot/helper/listeners/qbit_listener.py +++ b/bot/helper/listeners/qbit_listener.py @@ -31,11 +31,7 @@ stop_duplicate_check ) from ..task_utils.status_utils.qbit_status import QbittorrentStatus -from ..telegram_helper.message_utils import ( - auto_delete_message, - delete_links, - update_status_message -) +from ..telegram_helper.message_utils import update_status_message async def _remove_torrent(hash_, tag): @@ -154,10 +150,10 @@ async def _avg_speed_check(tor): LOGGER.info( f"Task is slower than minimum download speed: {task.listener.name} | {get_readable_file_size(dl_speed)}ps" ) - _on_download_error( + await _on_download_error( min_speed, tor - ) # type: ignore + ) @new_task diff --git a/bot/helper/listeners/task_listener.py b/bot/helper/listeners/task_listener.py index 1ad7cb9bbd18..8f404beecd9f 100644 --- a/bot/helper/listeners/task_listener.py +++ b/bot/helper/listeners/task_listener.py @@ -352,8 +352,6 @@ async def on_download_complete(self): await event.wait() # type: ignore if self.is_cancelled: return - async with queue_dict_lock: - non_queued_up.add(self.mid) LOGGER.info(f"Start from Queued/Upload: {self.name}") self.size = await get_path_size(up_dir) diff --git a/bot/helper/task_utils/download_utils/aria2_download.py b/bot/helper/task_utils/download_utils/aria2_download.py index be3327ca50bf..1c6532e1c15f 100644 --- a/bot/helper/task_utils/download_utils/aria2_download.py +++ b/bot/helper/task_utils/download_utils/aria2_download.py @@ -11,8 +11,6 @@ config_dict, aria2_options, aria2c_global, - non_queued_dl, - queue_dict_lock, ) from ...ext_utils.bot_utils import ( bt_selection_buttons, @@ -122,8 +120,6 @@ async def add_aria2c_download(listener, dpath, header, ratio, seed_time): await event.wait() # type: ignore if listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) async with task_dict_lock: task = task_dict[listener.mid] task.queued = False diff --git a/bot/helper/task_utils/download_utils/direct_downloader.py b/bot/helper/task_utils/download_utils/direct_downloader.py index 955126cc1894..02fdc93a0ff6 100644 --- a/bot/helper/task_utils/download_utils/direct_downloader.py +++ b/bot/helper/task_utils/download_utils/direct_downloader.py @@ -6,9 +6,7 @@ aria2_options, aria2c_global, task_dict, - task_dict_lock, - non_queued_dl, - queue_dict_lock, + task_dict_lock ) from ...ext_utils.bot_utils import sync_to_async from ...ext_utils.status_utils import get_readable_file_size @@ -82,8 +80,6 @@ async def add_direct_download(listener, path): await event.wait() # type: ignore if listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) a2c_opt = {**aria2_options} [ diff --git a/bot/helper/task_utils/download_utils/gd_download.py b/bot/helper/task_utils/download_utils/gd_download.py index ba542b7c3e48..8073b2150b51 100644 --- a/bot/helper/task_utils/download_utils/gd_download.py +++ b/bot/helper/task_utils/download_utils/gd_download.py @@ -2,8 +2,6 @@ from bot import ( LOGGER, - non_queued_dl, - queue_dict_lock, task_dict, task_dict_lock, ) @@ -90,8 +88,6 @@ async def add_gd_download(listener, path): await event.wait() # type: ignore if listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) drive = GoogleDriveDownload( listener, diff --git a/bot/helper/task_utils/download_utils/jd_download.py b/bot/helper/task_utils/download_utils/jd_download.py index d9284f06c187..a788db4bfa6f 100644 --- a/bot/helper/task_utils/download_utils/jd_download.py +++ b/bot/helper/task_utils/download_utils/jd_download.py @@ -23,15 +23,10 @@ LOGGER, jd_downloads, jd_lock, - non_queued_dl, - queue_dict_lock, task_dict, task_dict_lock, ) -from ...ext_utils.bot_utils import ( - new_task, - retry_function -) +from ...ext_utils.bot_utils import new_task from ...ext_utils.jdownloader_booter import jdownloader from ...ext_utils.task_manager import ( check_running_tasks, @@ -44,11 +39,10 @@ from ...telegram_helper.button_build import ButtonMaker from ...telegram_helper.message_utils import ( auto_delete_message, - delete_links, + delete_message, + edit_message, send_message, send_status_message, - edit_message, - delete_message, ) @@ -138,64 +132,67 @@ async def wait_for_configurations(self): async def get_online_packages(path, state="grabbing"): if state == "grabbing": - queued_downloads = await retry_function( - jdownloader.device.linkgrabber.query_packages, [{"saveTo": True}] # type: ignore - ) - return [qd["uuid"] for qd in queued_downloads if qd["saveTo"].startswith(path)] + queued_downloads = await jdownloader.device.linkgrabber.query_packages([{"saveTo": True}]) + return [ + qd["uuid"] + for qd in queued_downloads + if qd["saveTo"].startswith(path) + ] else: - download_packages = await retry_function( - jdownloader.device.downloads.query_packages, # type: ignore - [{"saveTo": True}], - ) - return [dl["uuid"] for dl in download_packages if dl["saveTo"].startswith(path)] + download_packages = await jdownloader.device.downloads.query_packages([{"saveTo": True}]) + return [ + dl["uuid"] + for dl in download_packages + if dl["saveTo"].startswith(path) + ] def trim_path(path): path_components = path.split("/") - trimmed_components = [ - component[:255] if len(component) > 255 else component + component[:255] + if len(component) > 255 + else component for component in path_components ] - return "/".join(trimmed_components) +async def get_jd_download_directory(): + res = await jdownloader.device.config.get( + "org.jdownloader.settings.GeneralSettings", + None, + "DefaultDownloadFolder" + ) + return f'/{res.strip("/")}/' + + async def add_jd_download(listener, path): try: async with jd_lock: - if jdownloader.device is None: + if not jdownloader.is_connected: raise MYJDException(jdownloader.error) - await jdownloader.check_jdownloader_state() - + default_path = await get_jd_download_directory() if not jd_downloads: - await retry_function(jdownloader.device.linkgrabber.clear_list) - if odl := await retry_function( - jdownloader.device.downloads.query_packages, - [{}] - ): - odl_list = [od["uuid"] for od in odl] - await retry_function( - jdownloader.device.downloads.remove_links, - package_ids=odl_list - ) - elif odl := await retry_function( - jdownloader.device.linkgrabber.query_packages, - [{}] - ): + await jdownloader.device.linkgrabber.clear_list() + if odl := await jdownloader.device.downloads.query_packages([{}]): + odl_list = [ + od["uuid"] + for od + in odl + ] + await jdownloader.device.downloads.remove_links(package_ids=odl_list) + elif odl := await jdownloader.device.linkgrabber.query_packages([{}]): if odl_list := [ od["uuid"] for od in odl if od.get( "saveTo", "" - ).startswith("/root/Downloads/") + ).startswith(default_path) ]: - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=odl_list - ) + await jdownloader.device.linkgrabber.remove_links(package_ids=odl_list) gid = token_urlsafe(12) jd_downloads[gid] = { @@ -210,14 +207,12 @@ async def add_jd_download(listener, path): ) as dlc: content = await dlc.read() content = b64encode(content) - await retry_function( - jdownloader.device.linkgrabber.add_container, + await jdownloader.device.linkgrabber.add_container( "DLC", f";base64,{content.decode()}" ) else: - await retry_function( - jdownloader.device.linkgrabber.add_links, + await jdownloader.device.linkgrabber.add_links( [ { "autoExtract": False, @@ -228,7 +223,7 @@ async def add_jd_download(listener, path): ) await sleep(1) - while await retry_function(jdownloader.device.linkgrabber.is_collecting): + while await jdownloader.device.linkgrabber.is_collecting(): pass start_time = time() online_packages = [] @@ -237,8 +232,7 @@ async def add_jd_download(listener, path): name = "" error = "" while (time() - start_time) < 60: - queued_downloads = await retry_function( - jdownloader.device.linkgrabber.query_packages, + queued_downloads = await jdownloader.device.linkgrabber.query_packages( [ { "bytesTotal": True, @@ -252,10 +246,7 @@ async def add_jd_download(listener, path): ) if not online_packages and corrupted_packages and error: - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=corrupted_packages - ) + await jdownloader.device.linkgrabber.remove_links(package_ids=corrupted_packages) raise MYJDException(error) for pack in queued_downloads: @@ -272,9 +263,9 @@ async def add_jd_download(listener, path): continue save_to = pack["saveTo"] if not name: - if save_to.startswith("/root/Downloads/"): + if save_to.startswith(default_path): name = save_to.replace( - "/root/Downloads/", + default_path, "", 1 ).split( @@ -313,23 +304,24 @@ async def add_jd_download(listener, path): 0 ) online_packages.append(pack["uuid"]) - if save_to.startswith("/root/Downloads/"): + if save_to.startswith(default_path): save_to = trim_path(save_to) - await retry_function( - jdownloader.device.linkgrabber.set_download_directory, + await jdownloader.device.linkgrabber.set_download_directory( save_to.replace( - "/root/Downloads", - path, + default_path, + f"{path}/", 1 ), [pack["uuid"]], ) if online_packages: - if listener.join and len(online_packages) > 1: + if ( + listener.join and + len(online_packages) > 1 + ): listener.name = listener.folder_name - await retry_function( - jdownloader.device.linkgrabber.move_to_new_package, + await jdownloader.device.linkgrabber.move_to_new_package( listener.name, f"{path}/{listener.name}", package_ids=online_packages, @@ -343,18 +335,14 @@ async def add_jd_download(listener, path): ) if corrupted_packages or online_packages: packages_to_remove = corrupted_packages + online_packages - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=packages_to_remove, - ) + await jdownloader.device.linkgrabber.remove_links(package_ids=packages_to_remove) raise MYJDException(error) jd_downloads[gid]["ids"] = online_packages corrupted_links = [] if remove_unknown: - links = await retry_function( - jdownloader.device.linkgrabber.query_links, + links = await jdownloader.device.linkgrabber.query_links( [{ "packageUUIDs": online_packages, "availability": True @@ -366,8 +354,7 @@ async def add_jd_download(listener, path): if link["availability"].lower() != "online" ] if corrupted_packages or corrupted_links: - await retry_function( - jdownloader.device.linkgrabber.remove_links, + await jdownloader.device.linkgrabber.remove_links( corrupted_links, corrupted_packages, ) @@ -379,10 +366,7 @@ async def add_jd_download(listener, path): button ) = await stop_duplicate_check(listener) if msg: - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=online_packages - ) + await jdownloader.device.linkgrabber.remove_links(package_ids=online_packages) await listener.on_download_error( msg, button @@ -396,10 +380,6 @@ async def add_jd_download(listener, path): is_jd=True ) if limit_exceeded: - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=online_packages - ) LOGGER.info(f"JDownloader Limit Exceeded: {listener.name} | {listener.size}") await listener.on_download_error(limit_exceeded) async with jd_lock: @@ -408,10 +388,7 @@ async def add_jd_download(listener, path): if listener.select: if not await JDownloaderHelper(listener).wait_for_configurations(): - await retry_function( - jdownloader.device.linkgrabber.remove_links, - package_ids=online_packages, - ) + await jdownloader.device.linkgrabber.remove_links(package_ids=online_packages) await listener.remove_from_same_dir() async with jd_lock: del jd_downloads[gid] @@ -419,7 +396,7 @@ async def add_jd_download(listener, path): else: online_packages = await get_online_packages(path) if not online_packages: - raise MYJDException("This Download have been removed manually!") + raise MYJDException("Select: This Download have been removed manually!") async with jd_lock: jd_downloads[gid]["ids"] = online_packages @@ -441,22 +418,15 @@ async def add_jd_download(listener, path): await event.wait() # type: ignore if listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) - await jdownloader.check_jdownloader_state() online_packages = await get_online_packages(path) if not online_packages: - raise MYJDException("This Download have been removed manually!") + raise MYJDException("Queue: This Download have been removed manually!") async with jd_lock: jd_downloads[gid]["ids"] = online_packages - await retry_function( - jdownloader.device.linkgrabber.move_to_downloadlist, - package_ids=online_packages, - ) - - await sleep(1) + await jdownloader.device.linkgrabber.move_to_downloadlist(package_ids=online_packages) + await sleep(0.5) online_packages = await get_online_packages( path, @@ -465,27 +435,21 @@ async def add_jd_download(listener, path): if not online_packages: online_packages = await get_online_packages(path) if not online_packages: - raise MYJDException("This Download have been removed manually!") - await retry_function( - jdownloader.device.linkgrabber.move_to_downloadlist, - package_ids=online_packages, - ) - await sleep(1) + raise MYJDException("Linkgrabber: This Download have been removed manually!") + await jdownloader.device.linkgrabber.move_to_downloadlist(package_ids=online_packages) + await sleep(0.5) online_packages = await get_online_packages( path, "down" ) if not online_packages: - raise MYJDException("This Download have been removed manually!") + raise MYJDException("Download List: This Download have been removed manually!") async with jd_lock: jd_downloads[gid]["status"] = "down" jd_downloads[gid]["ids"] = online_packages - await retry_function( - jdownloader.device.downloads.force_download, - package_ids=online_packages - ) + await jdownloader.device.downloads.force_download(package_ids=online_packages) async with task_dict_lock: task_dict[listener.mid] = JDownloaderStatus( @@ -512,3 +476,36 @@ async def add_jd_download(listener, path): finally: if await aiopath.exists(listener.link): await remove(listener.link) + + await sleep(2) + + links = await jdownloader.device.downloads.query_links( + [ + { + "packageUUIDs": online_packages, + "status": True, + } + ], + ) + links_to_remove = [] + force_download = False + for dlink in links: + if dlink["status"] == "Invalid download directory": + force_download = True + new_name, ext = dlink["name"].rsplit( + ".", + 1 + ) + new_name = new_name[: 250 - len(f".{ext}".encode())] + new_name = f"{new_name}.{ext}" + await jdownloader.device.downloads.rename_link( + dlink["uuid"], + new_name + ) + elif dlink["status"] == "HLS stream broken?": + links_to_remove.append(dlink["uuid"]) + + if links_to_remove: + await jdownloader.device.downloads.remove_links(links_to_remove) + if force_download: + await jdownloader.device.downloads.force_download(package_ids=online_packages) diff --git a/bot/helper/task_utils/download_utils/nzb_downloader.py b/bot/helper/task_utils/download_utils/nzb_downloader.py index 1ce04c6ce35d..da6e5512e100 100644 --- a/bot/helper/task_utils/download_utils/nzb_downloader.py +++ b/bot/helper/task_utils/download_utils/nzb_downloader.py @@ -14,8 +14,6 @@ from bot import ( LOGGER, config_dict, - non_queued_dl, - queue_dict_lock, sabnzbd_client, task_dict, task_dict_lock @@ -194,8 +192,6 @@ async def add_nzb(listener, path): await event.wait() # type: ignore if listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) async with task_dict_lock: task_dict[listener.mid].queued = False diff --git a/bot/helper/task_utils/download_utils/qbit_download.py b/bot/helper/task_utils/download_utils/qbit_download.py index 4826ea98c12c..c480516b6cfc 100644 --- a/bot/helper/task_utils/download_utils/qbit_download.py +++ b/bot/helper/task_utils/download_utils/qbit_download.py @@ -47,8 +47,7 @@ async def add_qb_torrent(listener, path, ratio, seed_time): is_paused=add_to_queue, tags=f"{listener.mid}", ratio_limit=ratio, - seeding_time_limit=seed_time, - headers={"user-agent": "Wget/1.12"} + seeding_time_limit=seed_time ) if op.lower() == "ok.": @@ -59,20 +58,18 @@ async def add_qb_torrent(listener, path, ratio, seed_time): if len(tor_info) == 0: start_time = time() - while True: + while (time() - start_time) <= 60: + if add_to_queue and event.is_set(): + add_to_queue = False tor_info = await sync_to_async( qbittorrent_client.torrents_info, tag=f"{listener.mid}" ) if len(tor_info) > 0: break - if time() - start_time > 60: - LOGGER.error("Download not started! This Torrent already added or unsupported/invalid link/file.") - await listener.on_download_error( - "Download not started!\nThis Torrent already added or unsupported/invalid link/file." - ) - return await sleep(1) + else: + raise Exception("Use torrent file or magnet link incase you have added direct link! Timed Out!") tor_info = tor_info[0] listener.name = tor_info.name @@ -146,20 +143,19 @@ async def add_qb_torrent(listener, path, ratio, seed_time): elif listener.multi <= 1: await send_status_message(listener.message) - if add_to_queue: - await event.wait() # type: ignore - if listener.is_cancelled: - return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) - async with task_dict_lock: - task_dict[listener.mid].queued = False - + if event is not None: + if not event.is_set(): + await event.wait() + if listener.is_cancelled: + return + async with task_dict_lock: + task_dict[listener.mid].queued = False + LOGGER.info(f"Start Queued Download from Qbittorrent: {tor_info.name} - Hash: {ext_hash}") await sync_to_async( qbittorrent_client.torrents_resume, torrent_hashes=ext_hash ) - LOGGER.info(f"Start Queued Download from Qbittorrent: {tor_info.name} - Hash: {ext_hash}") + except Exception as e: LOGGER.error(f"Qbittorrent download error: {e}") await listener.on_download_error(f"{e}") diff --git a/bot/helper/task_utils/download_utils/rclone_download.py b/bot/helper/task_utils/download_utils/rclone_download.py index ee9dbe4c0bb4..8b0836a1d530 100644 --- a/bot/helper/task_utils/download_utils/rclone_download.py +++ b/bot/helper/task_utils/download_utils/rclone_download.py @@ -5,8 +5,6 @@ from bot import ( LOGGER, - non_queued_dl, - queue_dict_lock, task_dict, task_dict_lock ) @@ -180,8 +178,6 @@ async def add_rclone_download(listener, path): await event.wait() # type: ignore if listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(listener.mid) RCTransfer = RcloneTransferHelper(listener) async with task_dict_lock: diff --git a/bot/helper/task_utils/download_utils/telegram_download.py b/bot/helper/task_utils/download_utils/telegram_download.py index 541430855b32..e42229147e5f 100644 --- a/bot/helper/task_utils/download_utils/telegram_download.py +++ b/bot/helper/task_utils/download_utils/telegram_download.py @@ -9,8 +9,6 @@ LOGGER, bot, config_dict, - non_queued_dl, - queue_dict_lock, task_dict, task_dict_lock, user @@ -198,8 +196,6 @@ async def add_download(self, message, path, session): await event.wait() # type: ignore if self._listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(self._listener.mid) await self._on_download_start(gid, add_to_queue) await self._download(message, path) diff --git a/bot/helper/task_utils/download_utils/yt_dlp_download.py b/bot/helper/task_utils/download_utils/yt_dlp_download.py index d6ab71ca4750..ad945e729c35 100644 --- a/bot/helper/task_utils/download_utils/yt_dlp_download.py +++ b/bot/helper/task_utils/download_utils/yt_dlp_download.py @@ -8,9 +8,7 @@ from bot import ( task_dict_lock, - task_dict, - non_queued_dl, - queue_dict_lock + task_dict ) from ...ext_utils.bot_utils import ( sync_to_async, @@ -28,11 +26,7 @@ stop_duplicate_check ) from ...task_utils.status_utils.queue_status import QueueStatus -from ...telegram_helper.message_utils import ( - auto_delete_message, - delete_links, - send_status_message -) +from ...telegram_helper.message_utils import send_status_message from ..status_utils.yt_dlp_download_status import YtDlpDownloadStatus LOGGER = getLogger(__name__) @@ -433,8 +427,6 @@ async def add_download(self, path, qual, playlist, options): await event.wait() # type: ignore if self._listener.is_cancelled: return - async with queue_dict_lock: - non_queued_dl.add(self._listener.mid) LOGGER.info(f"Start Queued Download from YT_DLP: {self._listener.name}") await self._on_download_start(True) diff --git a/bot/helper/task_utils/status_utils/jdownloader_status.py b/bot/helper/task_utils/status_utils/jdownloader_status.py index 19b9b99a102a..b9149ac67242 100644 --- a/bot/helper/task_utils/status_utils/jdownloader_status.py +++ b/bot/helper/task_utils/status_utils/jdownloader_status.py @@ -3,10 +3,7 @@ jd_lock, jd_downloads ) -from ...ext_utils.bot_utils import ( - retry_function, - async_to_sync -) +from ...ext_utils.bot_utils import async_to_sync from ...ext_utils.jdownloader_booter import jdownloader from ...ext_utils.status_utils import ( MirrorStatus, @@ -181,10 +178,7 @@ def gid(self): async def cancel_task(self): self.listener.is_cancelled = True LOGGER.info(f"Cancelling Download: {self.name()}") - await retry_function( - jdownloader.device.downloads.remove_links, # type: ignore - package_ids=jd_downloads[self._gid]["ids"], - ) + await jdownloader.device.downloads.remove_links(package_ids=jd_downloads[self._gid]["ids"]) async with jd_lock: del jd_downloads[self._gid] await self.listener.on_download_error("Download cancelled by user!") diff --git a/bot/modules/bot_settings.py b/bot/modules/bot_settings.py index 5097fdd56c2d..3171c382d96c 100644 --- a/bot/modules/bot_settings.py +++ b/bot/modules/bot_settings.py @@ -9,7 +9,6 @@ create_subprocess_exec, create_subprocess_shell, gather, - wait_for, ) from dotenv import load_dotenv from io import BytesIO @@ -45,7 +44,6 @@ global_extension_filter, index_urls, intervals, - jd_downloads, jd_lock, nzb_options, qbit_options, @@ -59,8 +57,7 @@ SetInterval, new_task, set_commands, - sync_to_async, - retry_function, + sync_to_async ) from ..helper.ext_utils.db_handler import database from ..helper.ext_utils.jdownloader_booter import jdownloader @@ -71,11 +68,11 @@ from ..helper.telegram_helper.button_build import ButtonMaker from ..helper.telegram_helper.filters import CustomFilters from ..helper.telegram_helper.message_utils import ( - send_message, - send_file, + delete_message, edit_message, + send_file, + send_message, update_status_message, - delete_message, ) from ..modules.rss import add_job from ..modules.torrent_search import initiate_search_tools @@ -689,7 +686,7 @@ async def edit_variable(message, pre_message, key): "JD_EMAIL", "JD_PASS" ]: - await jdownloader.initiate() + await jdownloader.boot() elif key == "RSS_DELAY": add_job() elif key == "USET_SERVERS": @@ -867,32 +864,11 @@ async def edit_nzb_server(message, pre_message, key, index=0): async def sync_jdownloader(): async with jd_lock: if ( - not config_dict["DATABASE_URL"] - or jdownloader.device is None + not config_dict["DATABASE_URL"] or + not jdownloader.is_connected ): return - try: - await wait_for( - retry_function(jdownloader.update_devices), - timeout=10 - ) - except: - is_connected = await jdownloader.jdconnect() - if not is_connected: - LOGGER.error(jdownloader.error) - return - isDeviceConnected = await jdownloader.connectToDevice() - if not isDeviceConnected: - LOGGER.error(jdownloader.error) - return await jdownloader.device.system.exit_jd() - is_connected = await jdownloader.jdconnect() - if not is_connected: - LOGGER.error(jdownloader.error) - return - isDeviceConnected = await jdownloader.connectToDevice() - if not isDeviceConnected: - LOGGER.error(jdownloader.error) if await aiopath.exists("cfg.zip"): await remove("cfg.zip") await ( @@ -1248,8 +1224,6 @@ async def edit_bot_settings(client, query): "JD_EMAIL", "JD_PASS" ]: - jdownloader.device = None - jdownloader.error = "JDownloader Credentials not provided!" await create_subprocess_exec( "pkill", "-9", diff --git a/bot/modules/clone.py b/bot/modules/clone.py index 41686b445971..3f382203895d 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -138,7 +138,23 @@ async def new_event(self): not self.link and (reply_to := self.message.reply_to_message) ): - self.link = reply_to.text.split("\n", 1)[0].strip() + try: + self.link = reply_to.text.split( + "\n", + 1 + )[0].strip() + except: + hmsg = await send_message( + self.message, + COMMAND_USAGE["clone"][0], + COMMAND_USAGE["clone"][1] + ) + await delete_message(self.pmsg) + await auto_delete_message( + self.message, + hmsg + ) + return await self.run_multi( input_list, diff --git a/bot/modules/rss.py b/bot/modules/rss.py index cfe274252ee8..5aaee51a6a91 100644 --- a/bot/modules/rss.py +++ b/bot/modules/rss.py @@ -52,6 +52,11 @@ rss_dict_lock = Lock() +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", +} async def rss_menu(event): user_id = event.from_user.id @@ -235,7 +240,9 @@ async def rss_sub(message): cmd = None stv = False try: - async with AsyncClient(verify=False) as client: + async with AsyncClient( + headers=headers, follow_redirects=True, timeout=60, verify=False + ) as client: res = await client.get(feed_link) html = res.text rss_d = feed_parse(html) @@ -500,7 +507,9 @@ async def rss_get(message): msg = await send_message( message, f"Getting the last {count} item(s) from {title}" ) - async with AsyncClient(verify=False) as client: + async with AsyncClient( + headers=headers, follow_redirects=True, timeout=60, verify=False + ) as client: res = await client.get(data["link"]) html = res.text rss_d = feed_parse(html) @@ -999,7 +1008,12 @@ async def rss_monitor(): tries = 0 while True: try: - async with AsyncClient(verify=False) as client: + async with AsyncClient( + headers=headers, + follow_redirects=True, + timeout=60, + verify=False, + ) as client: res = await client.get(data["link"]) html = res.text break diff --git a/config_sample.env b/config_sample.env index cc304f5c35d5..a4178e4b4b99 100644 --- a/config_sample.env +++ b/config_sample.env @@ -46,8 +46,7 @@ MEGA_EMAIL = "" MEGA_PASSWORD = "" # Sabnzbd -USENET_SERVERS = [{'name': 'main', 'host': '', 'port': 5126, 'timeout': 60, 'username': '', 'password': '', 'connections': 8, 'ssl': 1, 'ssl_verify': 2, 'ssl_ciphers': '', 'enable': 1, 'required': 0, 'optional': 0, 'retention': 0, 'send_group': 0, 'priority': 0}] - +USENET_SERVERS = "[{'name': 'main', 'host': '', 'port': 5126, 'timeout': 60, 'username': '', 'password': '', 'connections': 8, 'ssl': 1, 'ssl_verify': 2, 'ssl_ciphers': '', 'enable': 1, 'required': 0, 'optional': 0, 'retention': 0, 'send_group': 0, 'priority': 0}]" # Update UPSTREAM_REPO = "https://github.com/Dawn-India/Z-Mirror" UPSTREAM_BRANCH = "main" @@ -144,4 +143,4 @@ SEARCH_PLUGINS = '["https://raw.githubusercontent.com/qbittorrent/search-plugins "https://raw.githubusercontent.com/v1k45/1337x-qBittorrent-search-plugin/master/leetx.py", "https://raw.githubusercontent.com/nindogo/qbtSearchScripts/master/magnetdl.py", "https://raw.githubusercontent.com/msagca/qbittorrent_plugins/main/uniondht.py", - "https://raw.githubusercontent.com/khensolomon/leyts/master/yts.py"]' \ No newline at end of file + "https://raw.githubusercontent.com/khensolomon/leyts/master/yts.py"]' diff --git a/myjd/myjdapi.py b/myjd/myjdapi.py index 3de1825a5d2d..fec142ad92ef 100644 --- a/myjd/myjdapi.py +++ b/myjd/myjdapi.py @@ -1,11 +1,3 @@ -# -*- encoding: utf-8 -*- -from Crypto.Cipher import AES -from base64 import ( - b64encode, - b64decode -) -from hashlib import sha256 -from hmac import new from json import ( dumps, loads, @@ -16,29 +8,15 @@ RequestError ) from httpx import AsyncHTTPTransport -from time import time -from urllib.parse import quote from functools import wraps from .exception import ( MYJDApiException, MYJDConnectionException, - MYJDDecodeException, - MYJDDeviceNotFoundException, + MYJDDecodeException ) -BS = 16 - - -def PAD(s): - return s + ((BS - len(s) % BS) * chr(BS - len(s) % BS)).encode() - - -def UNPAD(s): - return s[: -s[-1]] - - class System: def __init__(self, device): self.device = device @@ -83,9 +61,6 @@ def __init__(self, device): self.url = "/config" async def list(self, params=None): - """ - :return: List - """ if params is None: return await self.device.action( f"{self.url}/list", @@ -95,23 +70,12 @@ async def list(self, params=None): return await self.device.action(f"{self.url}/list") async def listEnum(self, type): - """ - :return: List - """ return await self.device.action( f"{self.url}/listEnum", params=[type] ) async def get(self, interface_name, storage, key): - """ - :param interfaceName: a valid interface name from List - :type: str: - :param storage: 'null' to use default or 'cfg/' + interfaceName - :type: str: - :param key: a valid key from from List - :type: str: - """ params = [interface_name, storage, key] return await self.device.action( f"{self.url}/get", @@ -119,14 +83,6 @@ async def get(self, interface_name, storage, key): ) async def getDefault(self, interfaceName, storage, key): - """ - :param interfaceName: a valid interface name from List - :type: str: - :param storage: 'null' to use default or 'cfg/' + interfaceName - :type: str: - :param key: a valid key from from List - :type: str: - """ params = [interfaceName, storage, key] return await self.device.action( f"{self.url}/getDefault", @@ -134,23 +90,6 @@ async def getDefault(self, interfaceName, storage, key): ) async def query(self, params=None): - """ - :param params: A dictionary with options. The default dictionary is - configured so it returns you all config API entries with all details, but you - can put your own with your options. All the options available are this - ones: - { - "configInterface" : "", - "defaultValues" : True, - "description" : True, - "enumInfo" : True, - "includeExtensions": True, - "pattern" : "", - "values" : "" - } - :type: Dictionary - :rtype: List of dictionaries of this style, with more or less detail based on your options. - """ if params is None: params = [ { @@ -169,14 +108,6 @@ async def query(self, params=None): ) async def reset(self, interfaceName, storage, key): - """ - :param interfaceName: a valid interface name from List - :type: str: - :param storage: 'null' to use default or 'cfg/' + interfaceName - :type: str: - :param key: a valid key from from List - :type: str: - """ params = [ interfaceName, storage, @@ -188,16 +119,6 @@ async def reset(self, interfaceName, storage, key): ) async def set(self, interface_name, storage, key, value): - """ - :param interfaceName: a valid interface name from List - :type: str: - :param storage: 'null' to use default or 'cfg/' + interfaceName - :type: str: - :param key: a valid key from from List - :type: str: - :param value: a valid value for the given key (see type value from List) - :type: Object: - """ params = [ interface_name, storage, @@ -252,23 +173,6 @@ def __init__(self, device): self.url = "/extensions" async def list(self, params=None): - """ - :param params: A dictionary with options. The default dictionary is - configured so it returns you all available extensions, but you - can put your own with your options. All the options available are this - ones: - { - "configInterface" : True, - "description" : True, - "enabled" : True, - "iconKey" : True, - "name" : True, - "pattern" : "", - "installed" : True - } - :type: Dictionary - :rtype: List of dictionaries of this style, with more or less detail based on your options. - """ if params is None: params = [ { @@ -318,19 +222,9 @@ def __init__(self, device): self.url = "/linkgrabberv2" async def clear_list(self): - return await self.device.action( - f"{self.url}/clearList", - http_action="POST" - ) + return await self.device.action(f"{self.url}/clearList") async def move_to_downloadlist(self, link_ids=None, package_ids=None): - """ - Moves packages and/or links to download list. - - :param package_ids: Package UUID's. - :type: list of strings. - :param link_ids: Link UUID's. - """ if link_ids is None: link_ids = [] if package_ids is None: @@ -342,47 +236,6 @@ async def move_to_downloadlist(self, link_ids=None, package_ids=None): ) async def query_links(self, params=None): - """ - - Get the links in the linkcollector/linkgrabber - - :param params: A dictionary with options. The default dictionary is - configured so it returns you all the downloads with all details, but you - can put your own with your options. All the options available are this - ones: - { - "bytesTotal" : false, - "comment" : false, - "status" : false, - "enabled" : false, - "maxResults" : -1, - "startAt" : 0, - "packageUUIDs" : null, - "hosts" : false, - "url" : false, - "availability" : false, - "variantIcon" : false, - "variantName" : false, - "variantID" : false, - "variants" : false, - "priority" : false - } - :type: Dictionary - :rtype: List of dictionaries of this style, with more or less detail based on your options. - - [ { 'availability': 'ONLINE', - 'bytesTotal': 68548274, - 'enabled': True, - 'name': 'The Rick And Morty Theory - The Original Morty_ - ' - 'Cartoon Conspiracy (Ep. 74) @ChannelFred (192kbit).m4a', - 'packageUUID': 1450430888524, - 'url': 'youtubev2://DEMUX_M4A_192_720P_V4/d1NZf1w2BxQ/', - 'uuid': 1450430889576, - 'variant': { 'id': 'DEMUX_M4A_192_720P_V4', - 'name': '192kbit/s M4A-Audio'}, - 'variants': True - }, ... ] - """ if params is None: params = [ { @@ -415,21 +268,6 @@ async def cleanup( link_ids=None, package_ids=None ): - """ - Clean packages and/or links of the linkgrabber list. - Requires at least a package_ids or link_ids list, or both. - - :param package_ids: Package UUID's. - :type: list of strings. - :param link_ids: link UUID's. - :type: list of strings - :param action: Action to be done. Actions: DELETE_ALL, DELETE_DISABLED, DELETE_FAILED, DELETE_FINISHED, DELETE_OFFLINE, DELETE_DUPE, DELETE_MODE - :type: str: - :param mode: Mode to use. Modes: REMOVE_LINKS_AND_DELETE_FILES, REMOVE_LINKS_AND_RECYCLE_FILES, REMOVE_LINKS_ONLY - :type: str: - :param selection_type: Type of selection to use. Types: SELECTED, UNSELECTED, ALL, NONE - :type: str: - """ if link_ids is None: link_ids = [] if package_ids is None: @@ -449,15 +287,6 @@ async def cleanup( ) async def add_container(self, type_, content): - """ - Adds a container to Linkgrabber. - - :param type_: Type of container. - :type: string. - :param content: The container. - :type: string. - - """ params = [ type_, content @@ -468,16 +297,6 @@ async def add_container(self, type_, content): ) async def get_download_urls(self, link_ids, package_ids, url_display_type): - """ - Gets download urls from Linkgrabber. - - :param package_ids: Package UUID's. - :type: List of strings. - :param link_ids: link UUID's. - :type: List of strings - :param url_display_type: No clue. Not documented - :type: Dictionary - """ params = [ package_ids, link_ids, @@ -489,16 +308,6 @@ async def get_download_urls(self, link_ids, package_ids, url_display_type): ) async def set_priority(self, priority, link_ids, package_ids): - """ - Sets the priority of links or packages. - - :param package_ids: Package UUID's. - :type: list of strings. - :param link_ids: link UUID's. - :type: list of strings - :param priority: Priority to set. Priorities: HIGHEST, HIGHER, HIGH, DEFAULT, LOWER; - :type: str: - """ params = [ priority, link_ids, @@ -510,16 +319,6 @@ async def set_priority(self, priority, link_ids, package_ids): ) async def set_enabled(self, enable, link_ids, package_ids): - """ - Enable or disable packages. - - :param enable: Enable or disable package. - :type: boolean - :param link_ids: Links UUID. - :type: list of strings - :param package_ids: Packages UUID. - :type: list of strings. - """ params = [ enable, link_ids, @@ -531,37 +330,12 @@ async def set_enabled(self, enable, link_ids, package_ids): ) async def get_variants(self, params): - """ - Gets the variants of a url/download (not package), for example a youtube - link gives you a package with three downloads, the audio, the video and - a picture, and each of those downloads have different variants (audio - quality, video quality, and picture quality). - - :param params: List with the UUID of the download you want the variants. Ex: [232434] - :type: List - :rtype: Variants in a list with dictionaries like this one: [{'id': - 'M4A_256', 'name': '256kbit/s M4A-Audio'}, {'id': 'AAC_256', 'name': - '256kbit/s AAC-Audio'},.......] - """ return await self.device.action( f"{self.url}/getVariants", params ) async def add_links(self, params=None): - """ - Add links to the linkcollector - - { - "autostart" : false, - "links" : null, - "packageName" : null, - "extractPassword" : null, - "priority" : "DEFAULT", - "downloadPassword" : null, - "destinationFolder" : null - } - """ if params is None: params = [ { @@ -581,9 +355,6 @@ async def add_links(self, params=None): ) async def is_collecting(self): - """ - Boolean status query about the collecting process - """ return await self.device.action(f"{self.url}/isCollecting") async def set_download_directory(self, dir: str, package_ids: list): @@ -603,7 +374,6 @@ async def move_to_new_package( link_ids: list = None, package_ids: list = None ): - # Requires at least a link_ids or package_ids list, or both. if link_ids is None: link_ids = [] if package_ids is None: @@ -620,15 +390,6 @@ async def move_to_new_package( ) async def remove_links(self, link_ids=None, package_ids=None): - """ - Remove packages and/or links of the linkgrabber list. - Requires at least a link_ids or package_ids list, or both. - - :param link_ids: link UUID's. - :type: list of strings - :param package_ids: Package UUID's. - :type: list of strings. - """ if link_ids is None: link_ids = [] if package_ids is None: @@ -643,9 +404,6 @@ async def remove_links(self, link_ids=None, package_ids=None): ) async def rename_link(self, link_id, new_name): - """ - Renames files related with link_id - """ params = [ link_id, new_name @@ -659,9 +417,6 @@ async def get_package_count(self): return await self.device.action(f"{self.url}/getPackageCount") async def rename_package(self, package_id, new_name): - """ - Rename package name with package_id - """ params = [ package_id, new_name @@ -705,9 +460,6 @@ def __init__(self, device): self.url = "/downloadsV2" async def query_links(self, params=None): - """ - Get the links in the download list - """ if params is None: params = [ { @@ -774,21 +526,6 @@ async def cleanup( link_ids=None, package_ids=None ): - """ - Clean packages and/or links of the linkgrabber list. - Requires at least a package_ids or link_ids list, or both. - - :param package_ids: Package UUID's. - :type: list of strings. - :param link_ids: link UUID's. - :type: list of strings - :param action: Action to be done. Actions: DELETE_ALL, DELETE_DISABLED, DELETE_FAILED, DELETE_FINISHED, DELETE_OFFLINE, DELETE_DUPE, DELETE_MODE - :type: str: - :param mode: Mode to use. Modes: REMOVE_LINKS_AND_DELETE_FILES, REMOVE_LINKS_AND_RECYCLE_FILES, REMOVE_LINKS_ONLY - :type: str: - :param selection_type: Type of selection to use. Types: SELECTED, UNSELECTED, ALL, NONE - :type: str: - """ if link_ids is None: link_ids = [] if package_ids is None: @@ -808,16 +545,6 @@ async def cleanup( ) async def set_enabled(self, enable, link_ids, package_ids): - """ - Enable or disable packages. - - :param enable: Enable or disable package. - :type: boolean - :param link_ids: Links UUID. - :type: list of strings - :param package_ids: Packages UUID. - :type: list of strings. - """ params = [ enable, link_ids, @@ -855,16 +582,6 @@ async def set_dl_location(self, directory, package_ids=None): ) async def remove_links(self, link_ids=None, package_ids=None): - """ - Remove packages and/or links of the downloads list. - NOTE: For more specific removal, like deleting the files etc, use the /cleanup api. - Requires at least a link_ids or package_ids list, or both. - - :param link_ids: link UUID's. - :type: list of strings - :param package_ids: Package UUID's. - :type: list of strings. - """ if link_ids is None: link_ids = [] if package_ids is None: @@ -895,17 +612,27 @@ async def move_to_new_package( new_pkg_name, download_path ): - params = ( + params = [ link_ids, package_ids, new_pkg_name, download_path - ) + ] return await self.device.action( f"{self.url}/movetoNewPackage", params ) + async def rename_link(self, link_id: list, new_name: str): + params = [ + link_id, + new_name + ] + return await self.device.action( + f"{self.url}/renameLink", + params + ) + class Captcha: @@ -937,15 +664,8 @@ async def solve(self, captcha_id, solution): class Jddevice: - def __init__(self, jd, device_dict): - """This functions initializates the device instance. - It uses the provided dictionary to create the device. + def __init__(self, jd): - :param device_dict: Device dictionary - """ - self.name = device_dict["name"] - self.device_id = device_dict["id"] - self.device_type = device_dict["type"] self.myjd = jd self.config = Config(self) self.linkgrabber = Linkgrabber(self) @@ -955,131 +675,19 @@ def __init__(self, jd, device_dict): self.extensions = Extension(self) self.jd = Jd(self) self.system = System(self) - self.__direct_connection_info = None - self.__direct_connection_enabled = False - self.__direct_connection_cooldown = 0 - self.__direct_connection_consecutive_failures = 0 - - async def __refresh_direct_connections(self): - response = await self.myjd.request_api( - "/device/getDirectConnectionInfos", - "POST", - None, - self.__action_url() - ) - if ( - response is not None - and "data" in response - and "infos" in response["data"] - and len(response["data"]["infos"]) != 0 - ): - self.__update_direct_connections(response["data"]["infos"]) - - def __update_direct_connections(self, direct_info): - """ - Updates the direct_connections info keeping the order. - """ - tmp = [] - if self.__direct_connection_info is None: - tmp.extend( - { - "conn": conn, - "cooldown": 0 - } for conn in direct_info - ) - self.__direct_connection_info = tmp - return - # We remove old connections not available anymore. - for i in self.__direct_connection_info: - if i["conn"] not in direct_info: - tmp.remove(i) - else: - direct_info.remove(i["conn"]) - # We add new connections - tmp.extend( - { - "conn": conn, - "cooldown": 0 - } for conn in direct_info - ) - self.__direct_connection_info = tmp async def ping(self): return await self.action("/device/ping") - async def enable_direct_connection(self): - self.__direct_connection_enabled = True - await self.__refresh_direct_connections() - - def disable_direct_connection(self): - self.__direct_connection_enabled = False - self.__direct_connection_info = None - - async def action(self, path, params=(), http_action="POST"): - action_url = self.__action_url() - if ( - self.__direct_connection_enabled - and self.__direct_connection_info is not None - and time() >= self.__direct_connection_cooldown - ): - return await self.__direct_connect( - path, - http_action, - params, - action_url - ) - response = await self.myjd.request_api( - path, - http_action, - params, - action_url - ) - if response is None: - raise (MYJDConnectionException("No connection established\n")) - if ( - self.__direct_connection_enabled - and time() >= self.__direct_connection_cooldown - ): - await self.__refresh_direct_connections() - return response["data"] - - async def __direct_connect(self, path, http_action, params, action_url): - for conn in self.__direct_connection_info: # type: ignore - if time() > conn["cooldown"]: - connection = conn["conn"] - api = "http://" + connection["ip"] + ":" + str(connection["port"]) - response = await self.myjd.request_api( - path, - http_action, - params, - action_url, - api - ) - if response is not None: - self.__direct_connection_info.remove(conn) - self.__direct_connection_info.insert(0, conn) - self.__direct_connection_consecutive_failures = 0 - return response["data"] - else: - conn["cooldown"] = time() + 60 - self.__direct_connection_consecutive_failures += 1 - self.__direct_connection_cooldown = time() + ( - 60 * self.__direct_connection_consecutive_failures - ) + async def action(self, path, params=()): response = await self.myjd.request_api( path, - http_action, - params, - action_url + params ) if response is None: raise (MYJDConnectionException("No connection established\n")) - await self.__refresh_direct_connections() return response["data"] - def __action_url(self): - return f"/t_{self.myjd.get_session_token()}_{self.device_id}" - class clientSession(AsyncClient): @@ -1087,7 +695,7 @@ class clientSession(AsyncClient): async def request(self, method: str, url: str, **kwargs): kwargs.setdefault( "timeout", - 1.5 + 3 ) kwargs.setdefault( "follow_redirects", @@ -1103,19 +711,9 @@ async def request(self, method: str, url: str, **kwargs): class MyJdApi: def __init__(self): - self.__request_id = int(time() * 1000) - self.__api_url = "https://api.jdownloader.org" - self.__app_key = "zee" - self.__api_version = 1 - self.__devices = None - self.__login_secret = None - self.__device_secret = None - self.__session_token = None - self.__regain_token = None - self.__server_encryption_token = None - self.__device_encryption_token = None - self.__connected = False + self.__api_url = "http://127.0.0.1:3128" self._http_session = None + self.device = Jddevice(self) def _session(self): if self._http_session is not None: @@ -1125,412 +723,55 @@ def _session(self): retries=10, verify=False ) - self._http_session = clientSession(transport=transport) - self._http_session.verify = False - return self._http_session - def get_session_token(self): - return self.__session_token - - def is_connected(self): - """ - Indicates if there is a connection established. - """ - return self.__connected - - def set_app_key(self, app_key): - """ - Sets the APP Key. - """ - self.__app_key = app_key - - def __secret_create(self, email, password, domain): - """ - Calculates the login_secret and device_secret - - :param email: My.Jdownloader User email - :param password: My.Jdownloader User password - :param domain: The domain , if is for Server (login_secret) or Device (device_secret) - :return: secret hash - - """ - secret_hash = sha256() - secret_hash.update( - email.lower().encode("utf-8") - + password.encode("utf-8") - + domain.lower().encode("utf-8") - ) - return secret_hash.digest() - - def __update_encryption_tokens(self): - """ - Updates the server_encryption_token and device_encryption_token - - """ - if self.__server_encryption_token is None: - old_token = self.__login_secret - else: - old_token = self.__server_encryption_token - new_token = sha256() - new_token.update(old_token + bytearray.fromhex(self.__session_token)) # type: ignore - self.__server_encryption_token = new_token.digest() - new_token = sha256() - new_token.update(self.__device_secret + bytearray.fromhex(self.__session_token)) # type: ignore - self.__device_encryption_token = new_token.digest() - - def __signature_create(self, key, data): - """ - Calculates the signature for the data given a key. - - :param key: - :param data: - """ - signature = new( - key, - data.encode("utf-8"), - sha256 - ) - return signature.hexdigest() - - def __decrypt(self, secret_token, data): - """ - Decrypts the data from the server using the provided token - - :param secret_token: - :param data: - """ - init_vector = secret_token[: len(secret_token) // 2] - key = secret_token[len(secret_token) // 2 :] - decryptor = AES.new( - key, - AES.MODE_CBC, - init_vector - ) - return UNPAD(decryptor.decrypt(b64decode(data))) - - def __encrypt(self, secret_token, data): - """ - Encrypts the data from the server using the provided token - - :param secret_token: - :param data: - """ - data = PAD(data.encode("utf-8")) - init_vector = secret_token[: len(secret_token) // 2] - key = secret_token[len(secret_token) // 2 :] - encryptor = AES.new( - key, - AES.MODE_CBC, - init_vector - ) - encrypted_data = b64encode(encryptor.encrypt(data)) - return encrypted_data.decode("utf-8") - - def update_request_id(self): - """ - Updates Request_Id - """ - self.__request_id = int(time()) - - async def connect(self, email, password): - """Establish connection to api - - :param email: My.Jdownloader User email - :param password: My.Jdownloader User password - :returns: boolean -- True if succesful, False if there was any error. - - """ - self.__clean_resources() - self.__login_secret = self.__secret_create(email, password, "server") - self.__device_secret = self.__secret_create(email, password, "device") - response = await self.request_api( - "/my/connect", - "GET", - [ - ( - "email", - email - ), - ( - "appkey", - self.__app_key - ) - ] + async def request_api(self, path, params=None): + session = self._session() + params_request = ( + params + if params is not None + else [] ) - self.__connected = True - self.update_request_id() - self.__session_token = response["sessiontoken"] # type: ignore - self.__regain_token = response["regaintoken"] # type: ignore - self.__update_encryption_tokens() - return response - - async def reconnect(self): - """ - Reestablish connection to API. - - :returns: boolean -- True if successful, False if there was any error. - - """ - response = await self.request_api( - "/my/reconnect", - "GET", - [ - ( - "sessiontoken", - self.__session_token - ), - ( - "regaintoken", - self.__regain_token - ), - ], + params_request = {"params": params_request} + data = dumps(params_request) + data = data.replace( + '"null"', + "null" ) - self.update_request_id() - self.__session_token = response["sessiontoken"] # type: ignore - self.__regain_token = response["regaintoken"] # type: ignore - self.__update_encryption_tokens() - return response - - async def disconnect(self): - """ - Disconnects from API - - :returns: boolean -- True if successful, False if there was any error. - - """ - response = await self.request_api( - "/my/disconnect", - "GET", - [ - ( - "sessiontoken", - self.__session_token - ) - ] + data = data.replace( + "'null'", + "null" ) - self.__clean_resources() - if self._http_session is not None: - self._http_session = None - await self._http_session.aclose() # type: ignore - return response - - def __clean_resources(self): - self.update_request_id() - self.__login_secret = None - self.__device_secret = None - self.__session_token = None - self.__regain_token = None - self.__server_encryption_token = None - self.__device_encryption_token = None - self.__devices = None - self.__connected = False - - async def update_devices(self): - """ - Updates available devices. Use list_devices() to get the devices list. - - :returns: boolean -- True if successful, False if there was any error. - """ - response = await self.request_api( - "/my/listdevices", - "GET", - [ - ( - "sessiontoken", - self.__session_token - ) - ] - ) - self.update_request_id() - self.__devices = response["list"] # type: ignore - - def list_devices(self): - """ - Returns available devices. Use getDevices() to update the devices list. - Each device in the list is a dictionary like this example: - - { - 'name': 'Device', - 'id': 'af9d03a21ddb917492dc1af8a6427f11', - 'type': 'jd' - } - - :returns: list -- list of devices. - """ - return self.__devices - - def get_device(self, device_name=None, device_id=None): - """ - Returns a jddevice instance of the device - - :param deviceid: - """ - if not self.is_connected(): - raise (MYJDConnectionException("No connection established\n")) - if device_id is not None: - for device in self.__devices: # type: ignore - if device["id"] == device_id: - return Jddevice( - self, - device - ) - elif device_name is not None: - for device in self.__devices: # type: ignore - if device["name"] == device_name: - return Jddevice( - self, - device - ) - raise (MYJDDeviceNotFoundException("Device not found\n")) - - async def request_api( - self, - path, - http_method="GET", - params=None, - action=None, - api=None - ): - """ - Makes a request to the API to the 'path' using the 'http_method' with parameters,'params'. - Ex: - http_method=GET - params={"test":"test"} - post_params={"test2":"test2"} - action=True - This would make a request to "https://api.jdownloader.org" - """ - session = self._session() - if not api: - api = self.__api_url - data = None - if not self.is_connected() and path != "/my/connect": - raise (MYJDConnectionException("No connection established\n")) - if http_method == "GET": - query = [f"{path}?"] - if params is not None: - for param in params: - if param[0] != "encryptedLoginSecret": - query += [f"{param[0]}={quote(param[1])}"] - else: - query += [f"&{param[0]}={param[1]}"] - query += [f"rid={str(self.__request_id)}"] - if self.__server_encryption_token is None: - query += [ - "signature=" - + str( - self.__signature_create( - self.__login_secret, query[0] + "&".join(query[1:]) - ) - ) - ] - else: - query += [ - "signature=" - + str( - self.__signature_create( - self.__server_encryption_token, - query[0] + "&".join(query[1:]), - ) - ) - ] - query = query[0] + "&".join(query[1:]) + request_url = self.__api_url + path + try: res = await session.request( - http_method, - api + query - ) # type: ignore - encrypted_response = res.text - else: - params_request = [] - if params is not None: - for param in params: - if isinstance( - param, - ( - str, - list - ) - ): - params_request += [param] - elif isinstance( - param, - ( - dict, - bool - ) - ): - params_request += [dumps(param)] - else: - params_request += [str(param)] - params_request = { - "apiVer": self.__api_version, - "url": path, - "params": params_request, - "rid": self.__request_id, - } - data = dumps(params_request) - # Removing quotes around null elements. - data = data.replace( - '"null"', - "null" - ) - data = data.replace( - "'null'", - "null" + "POST", + request_url, + headers={"Content-Type": "application/json; charset=utf-8"}, + content=data, ) - encrypted_data = self.__encrypt( - self.__device_encryption_token, - data - ) - request_url = ( - api + action + path - if action is not None - else api + path - ) - try: - res = await session.request( - http_method, - request_url, - headers={"Content-Type": "application/aesjson-jd; charset=utf-8"}, - content=encrypted_data, - ) # type: ignore - encrypted_response = res.text - except RequestError: - return None + response = res.text + except RequestError: + return None if res.status_code != 200: try: - error_msg = loads(encrypted_response) - except JSONDecodeError: - try: - error_msg = loads( - self.__decrypt( - self.__device_encryption_token, - encrypted_response - ) - ) - except JSONDecodeError as exc: - raise MYJDDecodeException( - "Failed to decode response: {}", - encrypted_response - ) from exc + error_msg = loads(response) + except JSONDecodeError as exc: + raise MYJDDecodeException( + "Failed to decode response: {}", + response + ) from exc msg = ( "\n\tSOURCE: " + error_msg["src"] + "\n\tTYPE: " + error_msg["type"] + "\n------\nREQUEST_URL: " - + api - + ( - path - if http_method != "GET" - else "" - ) + + self.__api_url + + path ) - if http_method == "GET": - msg += query msg += "\n" if data is not None: msg += "DATA:\n" + data @@ -1541,25 +782,4 @@ async def request_api( msg ) ) - if action is None: - if not self.__server_encryption_token: - response = self.__decrypt( - self.__login_secret, - encrypted_response - ) - else: - response = self.__decrypt( - self.__server_encryption_token, - encrypted_response - ) - else: - response = self.__decrypt( - self.__device_encryption_token, - encrypted_response - ) - jsondata = loads(response.decode("utf-8")) - if jsondata["rid"] != self.__request_id: - self.update_request_id() - return None - self.update_request_id() - return jsondata + return loads(response) diff --git a/requirements.txt b/requirements.txt index ac98f4fce78e..d9d4023b25ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,6 @@ motor natsort pillow psutil -pycryptodome pymongo python-dotenv python-magic