4 GiB Support (#322)

* 4 GiB Support

.gitignore

@@ -173,3 +173,5 @@ reinforcement/*
 /ytdlbot/main.session
 /ytdlbot/tasks.session
 /ytdlbot/tasks.session-journal
+/ytdlbot/premium.session
+/dump.rdb

README.md

@@ -6,25 +6,14 @@ YouTube Download Bot🚀🎬⬇️
 
 This Telegram bot allows you to download videos from YouTube and other supported websites, including Instagram!
 
-<details> <summary>Deploy to heroku</summary>
-
-<a href="https://heroku.com/deploy"><img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy to Heroku"></a>
-
-If you are having trouble deploying, you can fork the project to your personal account and deploy it from there.
-
-**Starting November 28, 2022, free Heroku Dynos, free Heroku Postgres, and free Heroku Data for Redis® plans will no
-longer be available.**
-[Heroku Announcement](https://devcenter.heroku.com/articles/free-dyno-hours)
-</details>
-
 # Usage
 
 [https://t.me/benny_ytdlbot](https://t.me/benny_ytdlbot)
 
 Join Telegram Channel https://t.me/+OGRC8tp9-U9mZDZl for updates.
 
 Send link directly to the bot. Any
-Websites [supported by yt-dlp](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) will work t0o.
+Websites [supported by yt-dlp](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) will work too.
 
 # Limitations of my bot
 
@@ -50,6 +39,7 @@ own bot. See below instructions.
 10. subscriptions to YouTube Channels
 11. cache mechanism - download once for the same video.
 12. instagram posts
+13. 4 GiB file size support with Telegram Premium
 
 # Screenshots
 
@@ -115,6 +105,19 @@ One line command to run the bot
 docker run -e APP_ID=111 -e APP_HASH=111 -e TOKEN=370FXI bennythink/ytdlbot
 ```
 
+## Heroku
+
+<details> <summary>Deploy to heroku</summary>
+
+<a href="https://heroku.com/deploy"><img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy to Heroku"></a>
+
+If you are having trouble deploying, you can fork the project to your personal account and deploy it from there.
+
+**Starting November 28, 2022, free Heroku Dynos, free Heroku Postgres, and free Heroku Data for Redis® plans will no
+longer be available.**
+[Heroku Announcement](https://devcenter.heroku.com/articles/free-dyno-hours)
+</details>
+
 # Complete deployment guide for docker-compose
 
 * contains every functionality
@@ -172,6 +175,7 @@ You can configure all the following environment variables:
 * TMPFILE_PATH: tmpfile path(file download path)
 * TRONGRID_KEY: TronGrid key, better use your own key to avoid rate limit
 * TRON_MNEMONIC: Tron mnemonic, the default one is on nile testnet.
+* PREMIUM_USER: premium user ID, it can help you to download files larger than 2 GiB
 
 ## 3.2 Set up init data
 
@@ -248,8 +252,15 @@ On the other machine:
 docker-compose -f worker.yml up -d
 ```
 
-**⚠️ You should not publish Redis directly on the internet.
-Instead, you can use WireGuard to wrap it up for added security.**
+**⚠️ You should not publish Redis directly on the internet. ⚠️**
+
+### 4.4 4 GiB Support
+
+1. Subscribe to Telegram Premium
+2. Setup user id `PREMIUM_USER` in `ytdl.env`
+3. Create session file by running `python premium.py`
+4. Copy the session file `premium.session` to `data` directory
+5. `docker-compose up -d premium`
 
 ## kubernetes
 

docker-compose.yml

@@ -52,9 +52,11 @@ services:
     ports:
       - "127.0.0.1:15555:5555"
 
-  instagram:
+  premium:
     image: bennythink/ytdlbot
     env_file:
       - env/ytdl.env
     restart: always
-    command: [ "/usr/local/bin/python",  "/ytdlbot/ytdlbot/instagram.py" ]
+    volumes:
+      - ./data/premium.session:/ytdlbot/ytdlbot/premium.session
+    command: [ "/usr/local/bin/python",  "/ytdlbot/ytdlbot/premium.py" ]

ytdlbot/config.py

@@ -77,3 +77,9 @@
 # the default mnemonic is for nile testnet
 TRON_MNEMONIC = os.getenv("TRON_MNEMONIC", "cram floor today legend service drill pitch leaf car govern harvest soda")
 TRX_SIGNAL = signal("trx_received")
+
+PREMIUM_USER = int(os.getenv("PREMIUM_USER"))
+
+
+class FileTooBig(Exception):
+    pass

ytdlbot/database.py

@@ -25,7 +25,7 @@
 from beautifultable import BeautifulTable
 from influxdb import InfluxDBClient
 
-from config import MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS, IS_BACKUP_BOT
+from config import IS_BACKUP_BOT, MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS
 
 init_con = sqlite3.connect(":memory:", check_same_thread=False)
 

ytdlbot/downloader.py

@@ -27,7 +27,15 @@
 from pyrogram import types
 from tqdm import tqdm
 
-from config import AUDIO_FORMAT, ENABLE_ARIA2, ENABLE_FFMPEG, TG_MAX_SIZE, IPv6
+from config import (
+    AUDIO_FORMAT,
+    ENABLE_ARIA2,
+    ENABLE_FFMPEG,
+    PREMIUM_USER,
+    TG_MAX_SIZE,
+    FileTooBig,
+    IPv6,
+)
 from limit import Payment
 from utils import adjust_formats, apply_log_formatter, current_time, sizeof_fmt
 
@@ -92,7 +100,11 @@ def download_hook(d: dict, bot_msg):
         downloaded = d.get("downloaded_bytes", 0)
         total = d.get("total_bytes") or d.get("total_bytes_estimate", 0)
         if total > TG_MAX_SIZE:
-            raise Exception(f"Your download file size {sizeof_fmt(total)} is too large for Telegram.")
+            msg = f"Your download file size {sizeof_fmt(total)} is too large for Telegram."
+            if PREMIUM_USER:
+                raise FileTooBig(msg)
+            else:
+                raise Exception(msg)
 
         # percent = remove_bash_color(d.get("_percent_str", "N/A"))
         speed = remove_bash_color(d.get("_speed_str", "N/A"))
@@ -199,6 +211,8 @@ def ytdl_download(url: str, tempdir: str, bm, **kwargs) -> list:
                     ydl.download([url])
                 video_paths = list(pathlib.Path(tempdir).glob("*"))
                 break
+            except FileTooBig as e:
+                raise e
             except Exception:
                 error = traceback.format_exc()
                 logging.error("Download failed for %s - %s, try another way", format_, url)

ytdlbot/premium.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+# ytdlbot - premium.py
+# 2023-12-20  17:53
+
+import json
+import logging
+import pathlib
+import tempfile
+from unittest.mock import MagicMock
+
+import yt_dlp
+from pyrogram import Client, filters, types
+
+from config import APP_HASH, APP_ID, PYRO_WORKERS, TOKEN
+from limit import Payment
+from utils import apply_log_formatter
+
+apply_log_formatter()
+app = Client("premium", APP_ID, APP_HASH, workers=PYRO_WORKERS)
+
+BOT_ID = int(TOKEN.split(":")[0])
+
+
+@app.on_message(filters.user(BOT_ID) & filters.incoming)
+async def hello(client: Client, message: types.Message):
+    text = message.text
+    try:
+        data = json.loads(text)
+    except json.decoder.JSONDecodeError:
+        return
+    url = data["url"]
+    user_id = data["user_id"]
+
+    tempdir = tempfile.TemporaryDirectory(prefix="ytdl-")
+    output = pathlib.Path(tempdir.name, "%(title).70s.%(ext)s").as_posix()
+    ydl_opts = {"restrictfilenames": False, "quiet": True, "outtmpl": output}
+    formats = [
+        # webm , vp9 and av01 are not streamable on telegram, so we'll extract only mp4
+        "bestvideo[ext=mp4][vcodec!*=av01][vcodec!*=vp09]+bestaudio[ext=m4a]/bestvideo+bestaudio",
+        "bestvideo[vcodec^=avc]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best",
+        None,
+    ]
+
+    for f in formats:
+        ydl_opts["format"] = f
+        logging.info("Downloading BIG FILE for %s with format %s", url, f)
+        try:
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                ydl.download([url])
+            break
+        except Exception:
+            logging.error("Download failed for %s. Try other options...", url)
+
+    payment = Payment()
+    settings = payment.get_user_settings(user_id)
+    video_path = next(pathlib.Path(tempdir.name).glob("*"))
+    if settings[2] == "video" or isinstance(settings[2], MagicMock):
+        logging.info("Sending as video")
+        await client.send_video(
+            BOT_ID,
+            video_path.as_posix(),
+            caption="Powered by ytdlbot",
+            supports_streaming=True,
+            file_name=f"{user_id}.mp4",
+        )
+    elif settings[2] == "audio":
+        logging.info("Sending as audio")
+        await client.send_audio(
+            BOT_ID,
+            video_path.as_posix(),
+            caption="Powered by ytdlbot ",
+            file_name=f"{user_id}.mp3",
+        )
+    elif settings[2] == "document":
+        logging.info("Sending as document")
+        await client.send_document(
+            BOT_ID,
+            video_path.as_posix(),
+            caption="Powered by ytdlbot",
+            file_name=f"{user_id}.mp4",
+        )
+    else:
+        logging.error("Send type is not video or audio")
+
+    tempdir.cleanup()
+
+
+if __name__ == "__main__":
+    app.run()

ytdlbot/tasks.py

@@ -8,6 +8,7 @@
 __author__ = "Benny <[email protected]>"
 
 import asyncio
+import json
 import logging
 import os
 import pathlib
@@ -39,18 +40,19 @@
     ENABLE_CELERY,
     ENABLE_VIP,
     OWNER,
+    PREMIUM_USER,
     RATE_LIMIT,
     RCLONE_PATH,
     TMPFILE_PATH,
     WORKERS,
+    FileTooBig,
 )
 from constant import BotText
 from database import Redis
 from downloader import edit_text, tqdm_progress, upload_hook, ytdl_download
 from limit import Payment
 from utils import (
     apply_log_formatter,
-    auto_restart,
     customize_logger,
     get_metadata,
     get_revision,
@@ -81,7 +83,16 @@ def retrieve_message(chat_id: int, message_id: int) -> types.Message | Any:
 def ytdl_download_task(chat_id: int, message_id: int, url: str):
     logging.info("YouTube celery tasks started for %s", url)
     bot_msg = retrieve_message(chat_id, message_id)
-    ytdl_normal_download(bot, bot_msg, url)
+    try:
+        ytdl_normal_download(bot, bot_msg, url)
+    except FileTooBig as e:
+        # if you can go there, that means you have premium users set up
+        logging.warning("Seeking for help from premium user...")
+        bot_msg.edit_text(f"{e}\n\nPlease wait, I will try to send it as premium user")
+        data = {"url": url, "user_id": bot_msg.chat.id}
+        bot.send_message(PREMIUM_USER, json.dumps(data), disable_notification=True, disable_web_page_preview=True)
+    except Exception:
+        bot_msg.edit_text(f"Download failed!❌\n\n`{traceback.format_exc()[-2000:]}`", disable_web_page_preview=True)
     logging.info("YouTube celery tasks ended.")
 
 
@@ -140,12 +151,19 @@ def ytdl_download_entrance(client: Client, bot_msg: types.Message, url: str, mod
         redis.update_metrics("cache_miss")
         mode = mode or payment.get_user_settings(chat_id)[-1]
         if ENABLE_CELERY and mode in [None, "Celery"]:
+            # in celery mode, producer has lost control of this task.
             ytdl_download_task.delay(chat_id, bot_msg.id, url)
         else:
             ytdl_normal_download(client, bot_msg, url)
+    except FileTooBig as e:
+        logging.warning("Seeking for help from premium user...")
+        bot_msg.edit_text(f"{e}\n\nPlease wait, I will try to send it as premium user")
+        # this is only for normal node. Celery node will need to do it in celery tasks
+        data = {"url": url, "user_id": bot_msg.chat.id}
+        client.send_message(PREMIUM_USER, json.dumps(data), disable_notification=True, disable_web_page_preview=True)
     except Exception as e:
         logging.error("Failed to download %s, error: %s", url, e)
-        bot_msg.edit_text(f"Download failed!❌\n\n`{traceback.format_exc()[0:4000]}`", disable_web_page_preview=True)
+        bot_msg.edit_text(f"Download failed!❌\n\n`{traceback.format_exc()[-2000:]}`", disable_web_page_preview=True)
 
 
 def direct_download_entrance(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str):
@@ -484,7 +502,7 @@ def run_celery():
     threading.Thread(target=run_celery, daemon=True).start()
 
     scheduler = BackgroundScheduler(timezone="Europe/London")
-    scheduler.add_job(auto_restart, "interval", seconds=900)
+    # scheduler.add_job(auto_restart, "interval", seconds=900)
     scheduler.start()
 
     idle()

ytdlbot/utils.py

@@ -193,7 +193,7 @@ def auto_restart():
     if not os.path.exists(log_path):
         return
     with open(log_path) as f:
-        logs = "".join(tail_log(f, lines=100))
+        logs = "".join(tail_log(f, lines=50))
 
     det = Detector(logs)
     method_list = [getattr(det, func) for func in dir(det) if func.endswith("_detector")]

ytdlbot/ytdl_bot.py

@@ -40,6 +40,7 @@
     M3U8_SUPPORT,
     OWNER,
     PLAYLIST_SUPPORT,
+    PREMIUM_USER,
     PROVIDER_TOKEN,
     REQUIRED_MEMBERSHIP,
     TOKEN_PRICE,
@@ -345,6 +346,13 @@ def redeem_handler(client: Client, message: types.Message):
     message.reply_text(msg, quote=True)
 
 
+@app.on_message(filters.user(PREMIUM_USER) & filters.incoming & filters.caption)
+def premium_forward(client: Client, message: types.Message):
+    media = message.video or message.audio or message.document
+    target_user = media.file_name.split(".")[0]
+    client.forward_messages(target_user, message.chat.id, message.id)
+
+
 def generate_invoice(amount: int, title: str, description: str, payload: str):
     invoice = raw_types.input_media_invoice.InputMediaInvoice(
         invoice=raw_types.invoice.Invoice(