increase worker, separate redis for backup bot

requirements.txt

@@ -8,7 +8,7 @@ PyMySQL==1.1.0
 celery==5.3.6
 filetype==1.2.0
 flower==2.0.1
-psutil==5.9.6
+psutil==5.9.7
 influxdb==5.3.1
 beautifulsoup4==4.12.2
 fakeredis==2.20.1

ytdlbot/config.py

@@ -13,7 +13,7 @@
 
 # general settings
 WORKERS: int = int(os.getenv("WORKERS", 10))
-PYRO_WORKERS: int = int(os.getenv("PYRO_WORKERS", min(32, (os.cpu_count() or 0) + 10)))
+PYRO_WORKERS: int = int(os.getenv("PYRO_WORKERS", 100))
 APP_ID: int = int(os.getenv("APP_ID", 198214))
 APP_HASH = os.getenv("APP_HASH", "1234b90")
 TOKEN = os.getenv("TOKEN", "1234")
@@ -35,8 +35,12 @@
 REQUIRED_MEMBERSHIP: str = os.getenv("REQUIRED_MEMBERSHIP", "")
 
 # celery related
+IS_BACKUP_BOT = os.getenv("IS_BACKUP_BOT")
 ENABLE_CELERY = os.getenv("ENABLE_CELERY", False)
-BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/4")
+if IS_BACKUP_BOT:
+    BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/1")
+else:
+    BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/0")
 
 MYSQL_HOST = os.getenv("MYSQL_HOST", "mysql")
 MYSQL_USER = os.getenv("MYSQL_USER", "root")
@@ -53,7 +57,6 @@
 ENABLE_ARIA2 = os.getenv("ENABLE_ARIA2", False)
 
 RATE_LIMIT = os.getenv("RATE_LIMIT", 120)
-IS_BACKUP_BOT = os.getenv("IS_BACKUP_BOT")
 RCLONE_PATH = os.getenv("RCLONE")
 # This will set the value for the tmpfile path(download path) if it is set.
 # If TMPFILE is not set, it will return None and use system’s default temporary file path.

ytdlbot/database.py

@@ -25,7 +25,7 @@
 from beautifultable import BeautifulTable
 from influxdb import InfluxDBClient
 
-from config import MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS
+from config import MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS, IS_BACKUP_BOT
 
 init_con = sqlite3.connect(":memory:", check_same_thread=False)
 
@@ -71,12 +71,13 @@ def sub(sql):
 
 class Redis:
     def __init__(self):
+        db = 1 if IS_BACKUP_BOT else 0
         try:
-            self.r = redis.StrictRedis(host=REDIS, db=0, decode_responses=True)
+            self.r = redis.StrictRedis(host=REDIS, db=db, decode_responses=True)
             self.r.ping()
         except Exception:
-            logging.debug("Redis connection failed, using fake redis instead.")
-            self.r = fakeredis.FakeStrictRedis(host=REDIS, db=0, decode_responses=True)
+            logging.warning("Redis connection failed, using fake redis instead.")
+            self.r = fakeredis.FakeStrictRedis(host=REDIS, db=db, decode_responses=True)
 
         db_banner = "=" * 20 + "DB data" + "=" * 20
         quota_banner = "=" * 20 + "Celery" + "=" * 20
@@ -256,7 +257,7 @@ def __init__(self):
                 host=MYSQL_HOST, user=MYSQL_USER, passwd=MYSQL_PASS, db="ytdl", charset="utf8mb4"
             )
         except Exception:
-            logging.debug("MySQL connection failed, using fake mysql instead.")
+            logging.warning("MySQL connection failed, using fake mysql instead.")
             self.con = FakeMySQL()
 
         self.con.ping(reconnect=True)

ytdlbot/flower_tasks.py

@@ -11,4 +11,4 @@
 
 from config import BROKER
 
-app = Celery("tasks", broker=BROKER, timezone="Asia/Shanghai")
+app = Celery("tasks", broker=BROKER, timezone="Europe/London")

ytdlbot/utils.py

@@ -37,7 +37,6 @@ def apply_log_formatter():
 
 
 def customize_logger(logger: list):
-    apply_log_formatter()
     for log in logger:
         logging.getLogger(log).setLevel(level=logging.INFO)