From 513595cf7409a33355908a6aed89bb8e2497b09d Mon Sep 17 00:00:00 2001 From: 5hojib Date: Sat, 4 Jan 2025 14:59:57 +0600 Subject: [PATCH] LOG_CHAT_ID, improve mediainfo, metadata before ffmpeg, better logging --- bot/__init__.py | 38 +++++++++++++-- bot/core/config_manager.py | 1 + bot/helper/aeon_utils/gen_mediainfo.py | 19 -------- bot/helper/common.py | 20 +++++--- bot/helper/ext_utils/bot_utils.py | 6 ++- bot/helper/listeners/task_listener.py | 46 ++++++++++++------ bot/modules/mediainfo.py | 38 +++++++++++++-- bot/modules/mirror_leech.py | 11 +++-- bot/modules/services.py | 67 ++++++++++++++++++-------- bot/modules/ytdlp.py | 11 +++-- config_sample.py | 1 + update.py | 38 +++++++++++++-- 12 files changed, 212 insertions(+), 84 deletions(-) delete mode 100644 bot/helper/aeon_utils/gen_mediainfo.py diff --git a/bot/__init__.py b/bot/__init__.py index bbfd50b4d..800fe97b1 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -6,13 +6,18 @@ INFO, WARNING, FileHandler, + Formatter, + LogRecord, StreamHandler, basicConfig, + error, getLogger, + info, + warning, ) from socket import setdefaulttimeout from time import time - +from pytz import timezone from apscheduler.schedulers.asyncio import AsyncIOScheduler from aria2p import API as ariaAPI # noqa: N811 from aria2p import Client as ariaClient @@ -38,12 +43,35 @@ bot_loop = new_event_loop() set_event_loop(bot_loop) -basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[FileHandler("log.txt"), StreamHandler()], - level=INFO, +class CustomFormatter(Formatter): + def formatTime( # noqa: N802 + self, + record: LogRecord, + datefmt: str | None, + ) -> str: + dt: datetime = datetime.fromtimestamp( + record.created, + tz=timezone("Asia/Dhaka"), + ) + return dt.strftime(datefmt) + + def format(self, record: LogRecord) -> str: + return super().format(record).replace(record.levelname, record.levelname[:1]) + + +formatter = CustomFormatter( + "[%(asctime)s] %(levelname)s - %(message)s [%(module)s:%(lineno)d]", + datefmt="%d-%b %I:%M:%S %p", ) +file_handler = FileHandler("log.txt") +file_handler.setFormatter(formatter) + +stream_handler = StreamHandler() +stream_handler.setFormatter(formatter) + +basicConfig(handlers=[file_handler, stream_handler], level=INFO) + LOGGER = getLogger(__name__) intervals = {"status": {}, "qb": "", "stopAll": False} diff --git a/bot/core/config_manager.py b/bot/core/config_manager.py index 074d1906c..f2bf01a82 100644 --- a/bot/core/config_manager.py +++ b/bot/core/config_manager.py @@ -65,6 +65,7 @@ class Config: PAID_CHANNEL_LINK = "" DELETE_LINKS = False FSUB_IDS = "" + LOG_CHAT_ID = 0 @classmethod def get(cls, key): diff --git a/bot/helper/aeon_utils/gen_mediainfo.py b/bot/helper/aeon_utils/gen_mediainfo.py deleted file mode 100644 index afd7f9871..000000000 --- a/bot/helper/aeon_utils/gen_mediainfo.py +++ /dev/null @@ -1,19 +0,0 @@ -section_dict = {"General", "Video", "Audio", "Text", "Image"} - - -def parseinfo(out): - tc = "" - skip = False - for line in out.split("\n"): - if line.startswith("Menu"): - skip = True - elif any(line.startswith(section) for section in section_dict): - skip = False - if not line.startswith("General"): - tc += "
" - tc += f"
{line.replace('Text', 'Subtitle')}
"
-        if not skip:
-            key, sep, value = line.partition(":")
-            tc += f"{key.strip():<28}{sep} {value.strip()}\n"
-    tc += "

" - return tc diff --git a/bot/helper/common.py b/bot/helper/common.py index 580b0a1a5..b817de607 100644 --- a/bot/helper/common.py +++ b/bot/helper/common.py @@ -214,13 +214,21 @@ async def before_start(self): self.up_dest = self.user_dict["upload_paths"][self.up_dest] if self.ffmpeg_cmds and not isinstance(self.ffmpeg_cmds, list): - if self.user_dict.get("ffmpeg_cmds", None): - self.ffmpeg_cmds = self.user_dict["ffmpeg_cmds"].get( - self.ffmpeg_cmds, - None, - ) + ffmpeg_dict = self.user_dict["ffmpeg_cmds"] + self.ffmpeg_cmds = [ + value + for key in list(self.ffmpeg_cmds) + if key in ffmpeg_dict + for value in ffmpeg_dict[key] + ] elif "ffmpeg_cmds" not in self.user_dict and Config.FFMPEG_CMDS: - self.ffmpeg_cmds = Config.FFMPEG_CMDS.get(self.ffmpeg_cmds, None) + ffmpeg_dict = Config.FFMPEG_CMDS + self.ffmpeg_cmds = [ + value + for key in list(self.ffmpeg_cmds) + if key in ffmpeg_dict + for value in ffmpeg_dict[key] + ] else: self.ffmpeg_cmds = None if self.ffmpeg_cmds: diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index fe1bf0583..cd92c6827 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -149,7 +149,11 @@ def arg_parser(items, arg_base): break sub_list.append(items[j]) if sub_list: - arg_base[part] = " ".join(sub_list) + value = " ".join(sub_list) + if part == "-ff" and not value.strip().startswith("["): + arg_base[part].add(value) + else: + arg_base[part] = value i += len(sub_list) i += 1 if "link" in arg_base: diff --git a/bot/helper/listeners/task_listener.py b/bot/helper/listeners/task_listener.py index e088d79e8..d2e1d2af2 100644 --- a/bot/helper/listeners/task_listener.py +++ b/bot/helper/listeners/task_listener.py @@ -187,6 +187,20 @@ async def on_download_complete(self): self.subname = "" self.subsize = 0 + if self.metadata: + up_path = await self.proceed_metadata( + up_path, + gid + ) + if self.is_cancelled: + return + self.is_file = await aiopath.isfile(up_path) + up_dir, self.name = up_path.rsplit("/", 1) + self.size = await get_path_size(up_dir) + self.subproc = None + self.subname = "" + self.subsize = 0 + if self.ffmpeg_cmds: up_path = await self.proceed_ffmpeg( up_path, @@ -208,17 +222,6 @@ async def on_download_complete(self): self.is_file = await aiopath.isfile(up_path) self.name = up_path.rsplit("/", 1)[1] - if self.metadata: - up_path = await self.proceed_metadata(up_path, gid) - if self.is_cancelled: - return - self.is_file = await aiopath.isfile(up_path) - up_dir, self.name = up_path.rsplit("/", 1) - self.size = await get_path_size(up_dir) - self.subproc = None - self.subname = "" - self.subsize = 0 - if self.screen_shots: up_path = await self.generate_screenshots(up_path) if self.is_cancelled: @@ -350,6 +353,7 @@ async def on_upload_complete( ): await database.rm_complete_task(self.message.link) msg = f"Name: {escape(self.name)}\n\nSize: {get_readable_file_size(self.size)}" + done_msg = f"{self.tag}\nYour task is complete\nPlease check your inbox." LOGGER.info(f"Task Done: {self.name}") if self.is_leech: msg += f"\nTotal Files: {folders}" @@ -363,11 +367,22 @@ async def on_upload_complete( for index, (link, name) in enumerate(files.items(), start=1): fmsg += f"{index}. {name}\n" if len(fmsg.encode() + msg.encode()) > 4000: - await send_message(self.message, msg + fmsg) + await send_message(self.user_id, f"{msg}
{fmsg}
") + if Config.LOG_CHAT_ID: + await send_message( + Config.LOG_CHAT_ID, + f"{msg}
{fmsg}
", + ) await sleep(1) fmsg = "" if fmsg != "": - await send_message(self.message, msg + fmsg) + await send_message(self.user_id, f"{msg}
{fmsg}
") + if Config.LOG_CHAT_ID: + await send_message( + Config.LOG_CHAT_ID, + f"{msg}
{fmsg}
", + ) + await send_message(self.message, done_msg) else: msg += f"\n\nType: {mime_type}" if mime_type == "Folder": @@ -405,7 +420,10 @@ async def on_upload_complete( msg += f"\n\nPath: {rclone_path}" button = None msg += f"\n\ncc: {self.tag}" - await send_message(self.message, msg, button) + await send_message(self.user_id, msg, button) + if Config.LOG_CHAT_ID: + await send_message(Config.LOG_CHAT_ID, msg, button) + await send_message(self.message, done_msg) if self.seed: if self.new_dir: await clean_target(self.new_dir) diff --git a/bot/modules/mediainfo.py b/bot/modules/mediainfo.py index 9c3d73adf..59d8c6af3 100644 --- a/bot/modules/mediainfo.py +++ b/bot/modules/mediainfo.py @@ -12,7 +12,6 @@ from bot import LOGGER from bot.core.aeon_client import TgClient from bot.helper.aeon_utils.access_check import token_check -from bot.helper.aeon_utils.gen_mediainfo import parseinfo from bot.helper.ext_utils.bot_utils import cmd_exec from bot.helper.ext_utils.telegraph_helper import telegraph from bot.helper.telegram_helper.bot_commands import BotCommands @@ -24,6 +23,31 @@ send_message, ) +section_dict = {"General", "Video", "Audio", "Text", "Image"} + + +def parseinfo(out, file_size): + tc = "" + skip = False + file_size_line = f"File size : {file_size / (1024 * 1024):.2f} MiB" + + for line in out.split("\n"): + if line.startswith("Menu"): + skip = True + elif any(line.startswith(section) for section in section_dict): + skip = False + if not line.startswith("General"): + tc += "
" + tc += f"
{line.replace('Text', 'Subtitle')}
"
+        if not skip:
+            # Replace File size line
+            if line.startswith("File size"):
+                line = file_size_line
+            key, sep, value = line.partition(":")
+            tc += f"{key.strip():<28}{sep} {value.strip()}\n"
+    tc += "

" + return tc + async def gen_mediainfo(message, link=None, media=None, msg=None): temp_send = await send_message(message, "Generating MediaInfo...") @@ -32,6 +56,7 @@ async def gen_mediainfo(message, link=None, media=None, msg=None): if not await aiopath.isdir(path): await mkdir(path) + file_size = 0 if link: filename = re_search(".+/(.+)", link).group(1) des_path = ospath.join(path, filename) @@ -40,23 +65,28 @@ async def gen_mediainfo(message, link=None, media=None, msg=None): } async with aiohttp.ClientSession() as session: async with session.get(link, headers=headers) as response: + file_size = int(response.headers.get("Content-Length", 0)) async with aiopen(des_path, "wb") as f: async for chunk in response.content.iter_chunked(10000000): await f.write(chunk) break elif media: des_path = ospath.join(path, media.file_name) - if media.file_size <= 50000000: + file_size = media.file_size + if file_size <= 30000000: await msg.download(ospath.join(getcwd(), des_path)) else: - async for chunk in TgClient.bot.stream_media(media, limit=5): + async for chunk in TgClient.bot.stream_media(media, limit=3): async with aiopen(des_path, "ab") as f: await f.write(chunk) + # Get MediaInfo stdout, _, _ = await cmd_exec(ssplit(f'mediainfo "{des_path}"')) + + # Parse MediaInfo with updated file size tc = f"

{ospath.basename(des_path)}



" if stdout: - tc += parseinfo(stdout) + tc += parseinfo(stdout, file_size) except Exception as e: LOGGER.error(e) diff --git a/bot/modules/mirror_leech.py b/bot/modules/mirror_leech.py index 403cc0a41..279ed56f6 100644 --- a/bot/modules/mirror_leech.py +++ b/bot/modules/mirror_leech.py @@ -120,7 +120,7 @@ async def new_event(self): "-ns": "", "-md": "", "-tl": "", - "-ff": None, + "-ff": set(), } arg_parser(input_list[1:], args) @@ -168,10 +168,11 @@ async def new_event(self): self.multi = 0 try: - if args["-ff"].strip().startswith("["): - self.ffmpeg_cmds = eval(args["-ff"]) - else: - self.ffmpeg_cmds = args["-ff"] + if args["-ff"]: + if isinstance(args["-ff"], set): + self.ffmpeg_cmds = args["-ff"] + else: + self.ffmpeg_cmds = eval(args["-ff"]) except Exception as e: self.ffmpeg_cmds = None LOGGER.error(e) diff --git a/bot/modules/services.py b/bot/modules/services.py index 6c855a780..dc5ff7766 100644 --- a/bot/modules/services.py +++ b/bot/modules/services.py @@ -1,6 +1,8 @@ from time import time +from uuid import uuid4 from bot.helper.ext_utils.bot_utils import new_task +from bot.core.config_manager import Config from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.button_build import ButtonMaker from bot.helper.telegram_helper.filters import CustomFilters @@ -9,30 +11,55 @@ send_file, send_message, ) +from bot.helper.ext_utils.status_utils import get_readable_time @new_task -async def start(_, message): - buttons = ButtonMaker() - buttons.url_button( - "Repo", - "https://www.github.com/anasty17/mirror-leech-telegram-bot", - ) - buttons.url_button("Code Owner", "https://t.me/anas_tayyar") - reply_markup = buttons.build_menu(2) - if await CustomFilters.authorized(_, message): - start_string = f""" -This bot can mirror from links|tgfiles|torrents|rclone-cloud to any rclone cloud, Google Drive or to telegram. -Type /{BotCommands.HelpCommand} to get a list of available commands -""" - await send_message(message, start_string, reply_markup) +async def start(client, message): + if len(message.command) > 1 and message.command[1] == "private": + await delete_message(message) + elif len(message.command) > 1 and len(message.command[1]) == 36: + userid = message.from_user.id + input_token = message.command[1] + stored_token = await database.get_user_token(userid) + if stored_token is None: + return await send_message( + message, + "This token is not for you!\n\nPlease generate your own.", + ) + if input_token != stored_token: + return await send_message( + message, + "Invalid token.\n\nPlease generate a new one.", + ) + if userid not in user_data: + return await send_message( + message, + "This token is not yours!\n\nKindly generate your own.", + ) + data = user_data[userid] + if "token" not in data or data["token"] != input_token: + return await send_message( + message, + "This token has already been used!\n\nPlease get a new one.", + ) + token = str(uuid4()) + token_time = time() + data["token"] = token + data["time"] = token_time + user_data[userid].update(data) + await database.update_user_tdata(userid, token, token_time) + msg = "Your token has been successfully generated!\n\n" + msg += f'It will be valid for {get_readable_time(int(Config.TOKEN_TIMEOUT), True)}' + return await send_message(message, msg) + elif await CustomFilters.authorized(client, message): + help_command = f"/{BotCommands.HelpCommand}" + start_string = f"This bot can mirror all your links|files|torrents to Google Drive or any rclone cloud or to telegram.\nType {help_command} to get a list of available commands" + await send_message(message, start_string) else: - await send_message( - message, - "This bot can mirror from links|tgfiles|torrents|rclone-cloud to any rclone cloud, Google Drive or to telegram.\n\n⚠️ You Are not authorized user! Deploy your own mirror-leech bot", - reply_markup, - ) - + await send_message(message, "You are not a authorized user!") + await database.update_pm_users(message.from_user.id) + return None @new_task async def ping(_, message): diff --git a/bot/modules/ytdlp.py b/bot/modules/ytdlp.py index 9b90ed952..d55c92b52 100644 --- a/bot/modules/ytdlp.py +++ b/bot/modules/ytdlp.py @@ -333,7 +333,7 @@ async def new_event(self): "-ns": "", "-md": "", "-tl": "", - "-ff": None, + "-ff": set(), } arg_parser(input_list[1:], args) @@ -344,10 +344,11 @@ async def new_event(self): self.multi = 0 try: - if args["-ff"].strip().startswith("["): - self.ffmpeg_cmds = eval(args["-ff"]) - else: - self.ffmpeg_cmds = args["-ff"] + if args["-ff"]: + if isinstance(args["-ff"], set): + self.ffmpeg_cmds = args["-ff"] + else: + self.ffmpeg_cmds = eval(args["-ff"]) except Exception as e: self.ffmpeg_cmds = None LOGGER.error(e) diff --git a/config_sample.py b/config_sample.py index 15e70a661..f94d8035c 100644 --- a/config_sample.py +++ b/config_sample.py @@ -31,6 +31,7 @@ PAID_CHANNEL_LINK = "" SET_COMMANDS = True METADATA_KEY = "" +LOG_CHAT_ID = 0 # GDrive Tools GDRIVE_ID = "" diff --git a/update.py b/update.py index d76867336..b8609d3d4 100644 --- a/update.py +++ b/update.py @@ -3,9 +3,13 @@ ERROR, INFO, FileHandler, + Formatter, + LogRecord, StreamHandler, basicConfig, + error, getLogger, + info, ) from logging import ( error as log_error, @@ -16,7 +20,8 @@ from os import path, remove from subprocess import run as srun from sys import exit - +from pytz import timezone +from tzlocal import get_localzone from pymongo.mongo_client import MongoClient from pymongo.server_api import ServerApi @@ -29,12 +34,35 @@ if path.exists("rlog.txt"): remove("rlog.txt") -basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[FileHandler("log.txt"), StreamHandler()], - level=INFO, +class CustomFormatter(Formatter): + def formatTime( # noqa: N802 + self, + record: LogRecord, + datefmt: str | None, + ) -> str: + dt: datetime = datetime.fromtimestamp( + record.created, + tz=timezone("Asia/Dhaka"), + ) + return dt.strftime(datefmt) + + def format(self, record: LogRecord) -> str: + return super().format(record).replace(record.levelname, record.levelname[:1]) + + +formatter = CustomFormatter( + "[%(asctime)s] %(levelname)s - %(message)s [%(module)s:%(lineno)d]", + datefmt="%d-%b %I:%M:%S %p", ) +file_handler = FileHandler("log.txt") +file_handler.setFormatter(formatter) + +stream_handler = StreamHandler() +stream_handler.setFormatter(formatter) + +basicConfig(handlers=[file_handler, stream_handler], level=INFO) + settings = import_module("config") config_file = { key: value.strip() if isinstance(value, str) else value